diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..b0d3b3355 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: chrislusf +patreon: seaweedfs diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..2e7a327bd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" +- package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..b5899d12d --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +# What problem are we solving? + + + +# How are we solving the problem? + + +# Checks +- [ ] I have added unit tests if possible. +- [ ] I will add related wiki document changes and link to this PR after merging. diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml new file mode 100644 index 000000000..29b1cf6ab --- /dev/null +++ b/.github/workflows/binaries_dev.yml @@ -0,0 +1,124 @@ +name: "go: build dev binaries" + +on: + push: + branches: [ master ] + +permissions: + contents: read + +jobs: + + cleanup: + permissions: + contents: write # for mknejp/delete-release-assets to delete release assets + runs-on: ubuntu-latest + + steps: + + - name: Delete old release assets + uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1 + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-* + + build_dev_linux_windows: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + needs: cleanup + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux, windows] + goarch: [amd64] + + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} + + - name: Go Release Binaries Large Disk + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-normal-disk + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + build_dev_darwin: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + needs: build_dev_linux_windows + runs-on: ubuntu-latest + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} + + - name: Go Release Binaries Large Disk + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-normal-disk + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" diff --git a/.github/workflows/release_binaries.yml b/.github/workflows/binaries_release0.yml similarity index 66% rename from .github/workflows/release_binaries.yml rename to .github/workflows/binaries_release0.yml index 678f107d4..3a6cb734d 100644 --- a/.github/workflows/release_binaries.yml +++ b/.github/workflows/binaries_release0.yml @@ -1,44 +1,42 @@ # This is a basic workflow to help you get started with Actions -name: "go: build versioned binaries" +name: "go: build versioned binaries for windows" on: - release: - types: [created] + push: + tags: + - '*' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: # A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + jobs: - build: + + build-release-binaries_windows: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets runs-on: ubuntu-latest strategy: matrix: - goos: [linux, windows, darwin, freebsd, netbsd, openbsd] - goarch: [amd64, arm, arm64, 386] - exclude: - - goarch: arm - goos: darwin - - goarch: 386 - goos: darwin - - goarch: arm - goos: windows - - goarch: arm64 - goos: windows + goos: [windows] + goarch: [amd64] # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.20 + - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} goarch: ${{ matrix.goarch }} overwrite: true - pre_command: export CGO_ENABLED=0 + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 # build_flags: -tags 5BytesOffset # optional, default is ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` @@ -46,13 +44,13 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@v1.20 + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} goarch: ${{ matrix.goarch }} overwrite: true - pre_command: export CGO_ENABLED=0 + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 build_flags: -tags 5BytesOffset # optional, default is ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` diff --git a/.github/workflows/binaries_release1.yml b/.github/workflows/binaries_release1.yml new file mode 100644 index 000000000..d0a51ce8f --- /dev/null +++ b/.github/workflows/binaries_release1.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for linux" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_linux: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: [amd64, arm, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release2.yml b/.github/workflows/binaries_release2.yml new file mode 100644 index 000000000..09e8fc7ae --- /dev/null +++ b/.github/workflows/binaries_release2.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for darwin" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_darwin: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [darwin] + goarch: [amd64, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release3.yml b/.github/workflows/binaries_release3.yml new file mode 100644 index 000000000..c96a91cee --- /dev/null +++ b/.github/workflows/binaries_release3.yml @@ -0,0 +1,59 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for freebsd" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_freebsd: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [freebsd] + goarch: [amd64, arm, arm64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk" diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml new file mode 100644 index 000000000..71434e6a4 --- /dev/null +++ b/.github/workflows/binaries_release4.yml @@ -0,0 +1,60 @@ +# This is a basic workflow to help you get started with Actions + +name: "go: build versioned binaries for linux with all tags" + +on: + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +permissions: + contents: read + +jobs: + + build-release-binaries_linux: + permissions: + contents: write # for wangyoucao577/go-release-action to upload release assets + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: [amd64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - name: Go Release Binaries Normal Volume Size + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + build_flags: -tags elastic,ydb,gocdk + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + # build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full" + - name: Go Release Large Disk Binaries + uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + overwrite: true + pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 + build_flags: -tags 5BytesOffset,elastic,ydb,gocdk + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed + asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full_large_disk" diff --git a/.github/workflows/binary_test.yml b/.github/workflows/binary_test.yml deleted file mode 100644 index 8468555e3..000000000 --- a/.github/workflows/binary_test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "go: test building cross-platform binary" - -on: - pull_request: - workflow_dispatch: [] - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - strategy: - matrix: - goos: [linux, windows, darwin, freebsd, netbsd, openbsd] - goarch: [amd64, arm, arm64, 386] - exclude: - - goarch: arm - goos: darwin - - goarch: 386 - goos: darwin - - goarch: arm - goos: windows - - goarch: arm64 - goos: windows - - concurrency: - group: ${{ github.head_ref }}/binary_test/${{ matrix.goos }}/${{ matrix.goarch }} - cancel-in-progress: true - - steps: - - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ^1.13 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - name: Get dependencies - run: | - cd weed; go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - - - name: Build - run: cd weed; GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -v . diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 34e3a481b..000000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "chore: cleanup" - -on: - push: - branches: [ master ] - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - - steps: - - - name: Delete old release assets - uses: mknejp/delete-release-assets@v1 - with: - token: ${{ github.token }} - tag: dev - fail-if-no-assets: false - assets: | - weed-* diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..142e4e963 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,43 @@ +name: "Code Scanning - Action" + +on: + pull_request: + +jobs: + CodeQL-Build: + # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + runs-on: ubuntu-latest + + permissions: + # required for all workflows + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + # Override language selection by uncommenting this and choosing your languages + with: + languages: go + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below). + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml new file mode 100644 index 000000000..d8a2312ea --- /dev/null +++ b/.github/workflows/container_dev.yml @@ -0,0 +1,66 @@ +name: "docker: build dev containers" + +on: + push: + branches: [ master ] + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + + build-dev-containers: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + ghcr.io/chrislusf/seaweedfs + tags: | + type=raw,value=dev + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + with: + buildkitd-flags: "--debug" + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.go_build + platforms: linux/amd64, linux/arm64 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 1b952ae87..35dcea714 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -1,113 +1,63 @@ -name: "docker: build latest containers" +name: "docker: build latest container" on: push: - branches: - - master - workflow_dispatch: [] + tags: + - '*' + workflow_dispatch: {} -jobs: - build-latest: - runs-on: [ubuntu-latest] +permissions: + contents: read - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v3 - with: - images: | - chrislusf/seaweedfs - ghcr.io/chrislusf/seaweedfs - tags: | - type=raw,value=latest - labels: | - org.opencontainers.image.title=seaweedfs - org.opencontainers.image.vendor=Chris Lu - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - buildkitd-flags: "--debug" - - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GHCR - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build - uses: docker/build-push-action@v2 - with: - context: ./docker - push: ${{ github.event_name != 'pull_request' }} - file: ./docker/Dockerfile - platforms: linux/amd64, linux/arm, linux/arm64, linux/386 - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} +jobs: - build-dev: + build-latest-container: runs-on: [ubuntu-latest] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 - name: Docker meta id: docker_meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 with: images: | chrislusf/seaweedfs ghcr.io/chrislusf/seaweedfs tags: | - type=raw,value=dev + type=raw,value=latest labels: | org.opencontainers.image.title=seaweedfs org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 with: buildkitd-flags: "--debug" - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release.yml b/.github/workflows/container_release.yml deleted file mode 100644 index 789316df8..000000000 --- a/.github/workflows/container_release.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: "docker: build release containers" - -on: - push: - tags: - - '*' - workflow_dispatch: [] - -jobs: - build-default: - runs-on: [ubuntu-latest] - - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v3 - with: - images: | - chrislusf/seaweedfs - ghcr.io/chrislusf/seaweedfs - tags: | - type=ref,event=tag - flavor: | - latest=false - labels: | - org.opencontainers.image.title=seaweedfs - org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! - org.opencontainers.image.vendor=Chris Lu - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - buildkitd-flags: "--debug" - - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GHCR - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build - uses: docker/build-push-action@v2 - with: - context: ./docker - push: ${{ github.event_name != 'pull_request' }} - file: ./docker/Dockerfile.go_build - platforms: linux/amd64, linux/arm, linux/arm64, linux/386 - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-large: - runs-on: [ubuntu-latest] - - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v3 - with: - images: | - chrislusf/seaweedfs - ghcr.io/chrislusf/seaweedfs - tags: | - type=ref,event=tag,suffix=_large_disk - flavor: | - latest=false - labels: | - org.opencontainers.image.title=seaweedfs - org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! - org.opencontainers.image.vendor=Chris Lu - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - buildkitd-flags: "--debug" - - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GHCR - if: github.event_name != 'pull_request' - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build - uses: docker/build-push-action@v2 - with: - context: ./docker - push: ${{ github.event_name != 'pull_request' }} - file: ./docker/Dockerfile.go_build_large - platforms: linux/amd64, linux/arm, linux/arm64, linux/386 - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml new file mode 100644 index 000000000..1bcf768cd --- /dev/null +++ b/.github/workflows/container_release1.yml @@ -0,0 +1,57 @@ +name: "docker: build release containers for normal volume" + +on: + push: + tags: + - '*' + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build-default-release-container: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + tags: | + type=ref,event=tag + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.go_build + platforms: linux/amd64, linux/arm, linux/arm64, linux/386 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml new file mode 100644 index 000000000..c58bb2b40 --- /dev/null +++ b/.github/workflows/container_release2.yml @@ -0,0 +1,59 @@ +name: "docker: build release containers for large volume" + +on: + push: + tags: + - '*' + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + + build-large-release-container: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + tags: | + type=ref,event=tag,suffix=_large_disk + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.go_build + build-args: TAGS=5BytesOffset + platforms: linux/amd64, linux/arm, linux/arm64, linux/386 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml new file mode 100644 index 000000000..5ff6cd497 --- /dev/null +++ b/.github/workflows/container_release3.yml @@ -0,0 +1,58 @@ +name: "docker: build release containers for rocksdb" + +on: + push: + tags: + - '*' + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + + build-large-release-container_rocksdb: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + tags: | + type=ref,event=tag,suffix=_large_disk_rocksdb + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.rocksdb_large + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml new file mode 100644 index 000000000..1bba0ee9c --- /dev/null +++ b/.github/workflows/container_release4.yml @@ -0,0 +1,58 @@ +name: "docker: build release containers for all tags" + +on: + push: + tags: + - '*' + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build-default-release-container: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + tags: | + type=ref,event=tag,suffix=_full + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.go_build + build-args: TAGS=elastic,ydb,gocdk + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml new file mode 100644 index 000000000..181e86fc4 --- /dev/null +++ b/.github/workflows/container_release5.yml @@ -0,0 +1,58 @@ +name: "docker: build release containers for all tags and large volume" + +on: + push: + tags: + - '*' + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build-default-release-container: + runs-on: [ubuntu-latest] + + steps: + - + name: Checkout + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 + - + name: Docker meta + id: docker_meta + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3 + with: + images: | + chrislusf/seaweedfs + tags: | + type=ref,event=tag,suffix=_large_disk_full + flavor: | + latest=false + labels: | + org.opencontainers.image.title=seaweedfs + org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! + org.opencontainers.image.vendor=Chris Lu + - + name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build + uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2 + with: + context: ./docker + push: ${{ github.event_name != 'pull_request' }} + file: ./docker/Dockerfile.go_build + build-args: TAGS=5BytesOffset,elastic,ydb,gocdk + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/container_test.yml b/.github/workflows/container_test.yml deleted file mode 100644 index d86392af5..000000000 --- a/.github/workflows/container_test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: "docker: test building container images" - -on: - pull_request: - workflow_dispatch: [] - -concurrency: - group: ${{ github.head_ref }}/container_test - cancel-in-progress: true - -jobs: - build-test: - runs-on: [ubuntu-latest] - strategy: - matrix: - platform: [ linux ] - arch: [ amd64, arm, arm64, 386 ] - - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v3 - with: - images: | - chrislusf/seaweedfs - ghcr.io/chrislusf/seaweedfs - tags: | - type=raw,value=latest - labels: | - org.opencontainers.image.title=seaweedfs - org.opencontainers.image.vendor=Chris Lu - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - buildkitd-flags: "--debug" - - - name: Build - uses: docker/build-push-action@v2 - with: - context: ./docker - push: false - file: ./docker/Dockerfile - platforms: ${{ matrix.platform }}/${{ matrix.arch }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml new file mode 100644 index 000000000..b84b27d15 --- /dev/null +++ b/.github/workflows/depsreview.yml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748 + - name: 'Dependency Review' + uses: actions/dependency-review-action@1c59cdf2a9c7f29c90e8da32237eb04b81bad9f0 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 39b8347ac..b5c407315 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,4 +1,4 @@ -name: "go: test building binary" +name: "go: build binary" on: push: @@ -10,6 +10,9 @@ concurrency: group: ${{ github.head_ref }}/go cancel-in-progress: true +permissions: + contents: read + jobs: build: @@ -18,24 +21,20 @@ jobs: steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v2 with: go-version: ^1.13 id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 - name: Get dependencies run: | cd weed; go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - name: Build - run: cd weed; go build -v . + run: cd weed; go build -tags "elastic gocdk sqlite ydb" -v . - name: Test - run: cd weed; go test -v ./... + run: cd weed; go test -tags "elastic gocdk sqlite ydb" -v ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index ac04a85dc..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: "go: build dev binaries" - -on: - push: - branches: [ master ] - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - strategy: - matrix: - goos: [linux, windows, darwin, freebsd, netbsd, openbsd] - goarch: [amd64, arm, arm64, 386] - exclude: - - goarch: arm - goos: darwin - - goarch: 386 - goos: darwin - - goarch: arm - goos: windows - - goarch: arm64 - goos: windows - - steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - name: Wait for the deletion - uses: jakejarvis/wait-action@master - with: - time: '30s' - - - name: Set BUILD_TIME env - run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} - - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.20 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - pre_command: export CGO_ENABLED=0 - build_flags: -tags 5BytesOffset # optional, default is - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed-large-disk - asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.20 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - pre_command: export CGO_ENABLED=0 - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed - asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" diff --git a/.gitignore b/.gitignore index 671b01051..25a58bc67 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,8 @@ Temporary Items # Mongo Explorer plugin: # .idea/mongoSettings.xml +## vscode +.vscode ## File-based project format: *.ipr *.iws @@ -75,6 +77,8 @@ com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties +workspace/ + test_data build target diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a423a6ddd --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +BINARY = weed + +SOURCE_DIR = . + +all: install + +install: + cd weed; go install + +full_install: + cd weed; go install -tags "elastic gocdk sqlite ydb" + +test: + cd weed; go test -tags "elastic gocdk sqlite ydb" -v ./... diff --git a/README.md b/README.md index f07665651..a0fdd1492 100644 --- a/README.md +++ b/README.md @@ -31,16 +31,16 @@ Your support will be really appreciated by me and other supporters!

--> - ### Gold Sponsors -![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png) +- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com) --- - - [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest) - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS) +- [SeaweedFS on Telegram](https://t.me/Seaweedfs) +- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/) - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs) - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki) - [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf) @@ -51,12 +51,15 @@ Table of Contents ================= * [Quick Start](#quick-start) + * [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker) + * [Quick Start with Single Binary](#quick-start-with-single-binary) + * [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws) * [Introduction](#introduction) * [Features](#features) * [Additional Features](#additional-features) * [Filer Features](#filer-features) * [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store) -* [Architecture](#architecture) +* [Architecture](#Object-Store-Architecture) * [Compared to Other File Systems](#compared-to-other-file-systems) * [Compared to HDFS](#compared-to-hdfs) * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph) @@ -73,12 +76,15 @@ Table of Contents `docker run -p 8333:8333 chrislusf/seaweedfs server -s3` -## Quick Start with single binary ## +## Quick Start with Single Binary ## * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver=":9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it! +## Quick Start SeaweedFS S3 on AWS ## +* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc) + ## Introduction ## SeaweedFS is a simple and highly scalable distributed file system. There are two objectives: @@ -102,7 +108,7 @@ Also, SeaweedFS implements erasure coding with ideas from On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, -e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, etc. +e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc. For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, @@ -119,7 +125,7 @@ Faster and Cheaper than direct cloud storage! ## Additional Features ## * Can choose no replication or different replication levels, rack and data center aware. * Automatic master servers failover - no single point of failure (SPOF). -* Automatic Gzip compression depending on file mime type. +* Automatic Gzip compression depending on file MIME type. * Automatic compaction to reclaim disk space after deletion or update. * [Automatic entry TTL expiration][VolumeServerTTL]. * Any server with some disk spaces can add to the total storage space. @@ -147,6 +153,7 @@ Faster and Cheaper than direct cloud storage! * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data. * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB. * [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back. +* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive] ## Kubernetes ## * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/) @@ -170,6 +177,7 @@ Faster and Cheaper than direct cloud storage! [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication [KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store [CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture +[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage [Back to TOC](#table-of-contents) @@ -196,7 +204,7 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar ### Write File ### -To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url: +To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL: ``` > curl http://localhost:9333/dir/assign @@ -245,7 +253,7 @@ First look up the volume server's URLs by the file's volumeId: Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read. -Now you can take the public url, render the url or directly read from the volume server via url: +Now you can take the public URL, render the URL or directly read from the volume server via URL: ``` http://localhost:8080/3,01637037d6.jpg @@ -346,9 +354,9 @@ On each write request, the master server also generates a file key, which is a g ### Write and Read files ### -When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content. +When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the file. The client then contacts the volume node and POSTs the file content. -When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content. +When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content. Please see the example for details on the write-read process. @@ -402,7 +410,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files. * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached. -* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized. +* SeaweedFS Filer metadata store can be any well-known and proven data store, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd, YDB etc, and is easy to customize. * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc. | System | File Metadata | File Content Read| POSIX | REST API | Optimized for large number of small files | @@ -438,13 +446,13 @@ Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more com SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. -Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews. +Ceph, like SeaweedFS, is based on the object store RADOS. Ceph is rather complicated with mixed reviews. -Ceph uses CRUSH hashing to automatically manage the data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. Topology changes, such as adding new servers to increase capacity, will cause data migration with high IO cost to fit the CRUSH algorithm. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes are also as simple as it can be. +Ceph uses CRUSH hashing to automatically manage data placement, which is efficient to locate the data. But the data has to be placed according to the CRUSH algorithm. Any wrong configuration would cause data loss. Topology changes, such as adding new servers to increase capacity, will cause data migration with high IO cost to fit the CRUSH algorithm. SeaweedFS places data by assigning them to any writable volumes. If writes to one volume failed, just pick another volume to write. Adding more volumes is also as simple as it can be. SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read. -SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage. +SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, YDB, to manage file directories. These stores are proven, scalable, and easier to manage. | SeaweedFS | comparable to Ceph | advantage | | ------------- | ------------- | ---------------- | @@ -489,7 +497,7 @@ Step 1: install go on your machine and setup the environment by following the in https://golang.org/doc/install -make sure you set up your $GOPATH +make sure to define your $GOPATH Step 2: checkout this repo: @@ -499,7 +507,7 @@ git clone https://github.com/chrislusf/seaweedfs.git Step 3: download, compile, and install the project by executing the following command ```bash -make install +cd seaweedfs/weed && make install ``` Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory @@ -526,7 +534,7 @@ Write 1 million 1KB file: ``` Concurrency Level: 16 Time taken for tests: 66.753 seconds -Complete requests: 1048576 +Completed requests: 1048576 Failed requests: 0 Total transferred: 1106789009 bytes Requests per second: 15708.23 [#/sec] @@ -552,7 +560,7 @@ Randomly read 1 million files: ``` Concurrency Level: 16 Time taken for tests: 22.301 seconds -Complete requests: 1048576 +Completed requests: 1048576 Failed requests: 0 Total transferred: 1106812873 bytes Requests per second: 47019.38 [#/sec] diff --git a/backers.md b/backers.md index dd8c69e67..6f2d83a7c 100644 --- a/backers.md +++ b/backers.md @@ -5,12 +5,16 @@

Generous Backers ($50+)

-- [4Sight Imaging](https://www.4sightimaging.com/) - [Evercam Camera Management Software](https://evercam.io/) -- [Admiral](https://getadmiral.com)

Backers

- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/) - [Haravan - Ecommerce Platform](https://www.haravan.com) - PeterCxy - Creator of Shelter App +- [Hive Games](https://playhive.com/) +- Flowm +- Yoni Nakache +- Catalin Constantin +- MingLi Yuan +- Leroy van Logchem diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index d2c365c29..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM alpine - -# 'latest' or 'dev' -ARG RELEASE=latest - -RUN \ - ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \ - elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \ - elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \ - elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \ - elif [ $(uname -m) == "armv6l" ]; then echo "arm"; \ - elif [ $(uname -m) == "s390x" ]; then echo "s390x"; \ - elif [ $(uname -m) == "ppc64le" ]; then echo "ppc64le"; fi;) && \ - echo "Building for $ARCH" 1>&2 && \ - SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \ - SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \ - SUPERCRONIC=supercronic-linux-$ARCH && \ - # Install SeaweedFS and Supercronic ( for cron job mode ) - apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \ - apk add fuse && \ - wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz" | head -n 1) && \ - tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \ - curl -fsSLO "$SUPERCRONIC_URL" && \ - echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ - chmod +x "$SUPERCRONIC" && \ - mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \ - ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \ - apk del build-dependencies && \ - rm -rf /tmp/* - -# volume server gprc port -EXPOSE 18080 -# volume server http port -EXPOSE 8080 -# filer server gprc port -EXPOSE 18888 -# filer server http port -EXPOSE 8888 -# master server shared gprc port -EXPOSE 19333 -# master server shared http port -EXPOSE 9333 -# s3 server http port -EXPOSE 8333 -# webdav server http port -EXPOSE 7333 - -RUN mkdir -p /data/filerldb2 - -VOLUME /data - -COPY filer.toml /etc/seaweedfs/filer.toml -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.gccgo_build b/docker/Dockerfile.gccgo_build index e42a593cb..90cdf352f 100644 --- a/docker/Dockerfile.gccgo_build +++ b/docker/Dockerfile.gccgo_build @@ -37,6 +37,7 @@ EXPOSE 7333 RUN mkdir -p /data/filerldb2 VOLUME /data +WORKDIR /data RUN chmod +x /entrypoint.sh diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index 3a62cf5a3..c917ec556 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -1,12 +1,13 @@ -FROM amd64/golang:1.17-alpine as builder +FROM golang:1.18-alpine as builder RUN apk add git g++ fuse RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs ARG BRANCH=${BRANCH:-master} +ARG TAGS RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ - && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" + && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" FROM alpine AS final LABEL author="Chris Lu" @@ -36,6 +37,7 @@ EXPOSE 7333 RUN mkdir -p /data/filerldb2 VOLUME /data +WORKDIR /data RUN chmod +x /entrypoint.sh diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local index 0a85c56f0..947edffda 100644 --- a/docker/Dockerfile.local +++ b/docker/Dockerfile.local @@ -26,6 +26,7 @@ EXPOSE 7333 RUN mkdir -p /data/filerldb2 VOLUME /data +WORKDIR /data RUN chmod +x /entrypoint.sh diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.rocksdb_large similarity index 57% rename from docker/Dockerfile.go_build_large rename to docker/Dockerfile.rocksdb_large index b0433257c..0025eb116 100644 --- a/docker/Dockerfile.go_build_large +++ b/docker/Dockerfile.rocksdb_large @@ -1,20 +1,37 @@ -FROM amd64/golang:1.17-alpine as builder -RUN apk add git g++ fuse +FROM golang:1.18-buster as builder + +RUN apt-get update +RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev + +ENV ROCKSDB_VERSION v7.2.2 + +# build RocksDB +RUN cd /tmp && \ + git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ + cd rocksdb && \ + PORTABLE=1 make static_lib && \ + make install-static + +ENV CGO_CFLAGS "-I/tmp/rocksdb/include" +ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" + +# build SeaweedFS RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs ARG BRANCH=${BRANCH:-master} RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ - && CGO_ENABLED=0 go install -tags 5BytesOffset -ldflags "-extldflags -static ${LDFLAGS}" + && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" + FROM alpine AS final LABEL author="Chris Lu" COPY --from=builder /go/bin/weed /usr/bin/ RUN mkdir -p /etc/seaweedfs -COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh -RUN apk add fuse # for weed mount +RUN apk add fuse snappy gflags # volume server gprc port EXPOSE 18080 @@ -33,10 +50,12 @@ EXPOSE 8333 # webdav server http port EXPOSE 7333 -RUN mkdir -p /data/filerldb2 +RUN mkdir -p /data/filer_rocksdb VOLUME /data +WORKDIR /data + RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.s3tests b/docker/Dockerfile.s3tests index 5b6d762de..e55ebadf9 100644 --- a/docker/Dockerfile.s3tests +++ b/docker/Dockerfile.s3tests @@ -28,4 +28,4 @@ ENV \ S3TEST_CONF="/s3test.conf" ENTRYPOINT ["/bin/bash", "-c"] -CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] \ No newline at end of file +CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] \ No newline at end of file diff --git a/docker/Makefile b/docker/Makefile index 8efc0ded2..3afea17c1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -7,12 +7,21 @@ gen: dev binary: export SWCOMMIT=$(shell git rev-parse --short HEAD) export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)" - cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ + cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ build: binary docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local . rm ./weed +go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset + docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build . + +go_build_large_disk: + docker build --build-arg TAGS=large_disk --no-cache -t chrislusf/seaweedfs:large_disk -f Dockerfile.go_build . + +build_rocksdb: + docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large . + s3tests_build: docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests . @@ -40,18 +49,38 @@ dev_registry: build dev_replicate: build docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up +dev_auditlog: build + docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up + +dev_nextcloud: build + docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up + cluster: build docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up 2clusters: build docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up +2mount: build + docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up + +hashicorp_raft: build + docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up + s3tests: build s3tests_build docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up filer_etcd: build docker stack deploy -c compose/swarm-etcd.yml fs +test_etcd: build + docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up + +test_ydb: tags = ydb +test_ydb: build + export + docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up + clean: rm ./weed @@ -65,4 +94,4 @@ certstrap: certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true - certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true \ No newline at end of file + certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true diff --git a/docker/README.md b/docker/README.md index d6e1f4928..524f5d4d0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -37,3 +37,8 @@ docker buildx build --pull --push --platform linux/386,linux/amd64,linux/arm64,l docker buildx stop $BUILDER ``` +## Minio debuging +``` +mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1 +mc admin trace --all --verbose local +``` \ No newline at end of file diff --git a/docker/compose/fluent.json b/docker/compose/fluent.json new file mode 100644 index 000000000..8b50b0f72 --- /dev/null +++ b/docker/compose/fluent.json @@ -0,0 +1,4 @@ +{ + "fluent_port": 24224, + "fluent_host": "fluent" +} \ No newline at end of file diff --git a/docker/compose/local-auditlog-compose.yml b/docker/compose/local-auditlog-compose.yml new file mode 100644 index 000000000..39c997448 --- /dev/null +++ b/docker/compose/local-auditlog-compose.yml @@ -0,0 +1,36 @@ +version: '2' + +services: + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + - 9333:9333 + - 19333:19333 + - 8084:8080 + - 18084:18080 + - 8888:8888 + - 18888:18888 + - 8000:8000 + command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1" + volumes: + - ./fluent.json:/etc/seaweedfs/fluent.json + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - fluent + fluent: + image: fluent/fluentd:v1.14 + ports: + - 24224:24224 + #s3tests: + # image: chrislusf/ceph-s3-tests:local + # volumes: + # - ./s3tests.conf:/opt/s3-tests/s3tests.conf + # environment: + # S3TEST_CONF: "s3tests.conf" + # NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" + # NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" + # NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" + # depends_on: + # - s3 + # - fluent \ No newline at end of file diff --git a/docker/compose/local-cluster-compose.yml b/docker/compose/local-cluster-compose.yml index 82095ae18..f781244ab 100644 --- a/docker/compose/local-cluster-compose.yml +++ b/docker/compose/local-cluster-compose.yml @@ -6,25 +6,37 @@ services: ports: - 9333:9333 - 19333:19333 - command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1" + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 master1: image: chrislusf/seaweedfs:local ports: - 9334:9334 - 19334:19334 - command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2" + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 master2: image: chrislusf/seaweedfs:local ports: - 9335:9335 - 19335:19335 - command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3" + command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp" + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 volume1: image: chrislusf/seaweedfs:local ports: - 8080:8080 - 18080:18080 - command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1' + command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1' depends_on: - master0 - master1 @@ -34,7 +46,7 @@ services: ports: - 8082:8082 - 18082:18082 - command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1' + command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1' depends_on: - master0 - master1 @@ -44,7 +56,7 @@ services: ports: - 8083:8083 - 18083:18083 - command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' + command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' depends_on: - master0 - master1 @@ -54,7 +66,8 @@ services: ports: - 8888:8888 - 18888:18888 - command: 'filer -master="master0:9333,master1:9334,master2:9335"' + - 8111:8111 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"' depends_on: - master0 - master1 @@ -65,7 +78,7 @@ services: image: chrislusf/seaweedfs:local ports: - 8333:8333 - command: 's3 -filer="filer:8888"' + command: '-v=9 s3 -filer="filer:8888"' depends_on: - master0 - master1 diff --git a/docker/compose/local-hashicorp-raft-compose.yml b/docker/compose/local-hashicorp-raft-compose.yml new file mode 100644 index 000000000..14b5eb57a --- /dev/null +++ b/docker/compose/local-hashicorp-raft-compose.yml @@ -0,0 +1,89 @@ +version: '2' + +services: + master0: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data" + volumes: + - ./master/0:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master1: + image: chrislusf/seaweedfs:local + ports: + - 9334:9334 + - 19334:19334 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data" + volumes: + - ./master/1:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + master2: + image: chrislusf/seaweedfs:local + ports: + - 9335:9335 + - 19335:19335 + command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data" + volumes: + - ./master/2:/data + environment: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 2 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + volume1: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1' + depends_on: + - master0 + - master1 + volume2: + image: chrislusf/seaweedfs:local + ports: + - 8082:8082 + - 18082:18082 + command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1' + depends_on: + - master0 + - master1 + volume3: + image: chrislusf/seaweedfs:local + ports: + - 8083:8083 + - 18083:18083 + command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1' + depends_on: + - master0 + - master1 + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8111:8111 + command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"' + depends_on: + - master0 + - master1 + - volume1 + - volume2 + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"' + depends_on: + - master0 + - master1 + - volume1 + - volume2 + - filer \ No newline at end of file diff --git a/docker/compose/local-nextcloud-compose.yml b/docker/compose/local-nextcloud-compose.yml new file mode 100644 index 000000000..80c3fca53 --- /dev/null +++ b/docker/compose/local-nextcloud-compose.yml @@ -0,0 +1,44 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -mserver=master:9333 -port=8080 -ip=volume" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3' + depends_on: + - master + - volume + nextcloud: + image: nextcloud:23.0.5-apache + environment: + - OBJECTSTORE_S3_HOST=s3 + - OBJECTSTORE_S3_BUCKET=nextcloud + - OBJECTSTORE_S3_KEY=some_access_key1 + - OBJECTSTORE_S3_SECRET=some_secret_key1 + - OBJECTSTORE_S3_PORT=8333 + - OBJECTSTORE_S3_SSL=false + - OBJECTSTORE_S3_USEPATH_STYLE=true + - SQLITE_DATABASE=nextcloud + - NEXTCLOUD_ADMIN_USER=admin + - NEXTCLOUD_ADMIN_PASSWORD=admin + ports: + - 80:80 + depends_on: + - s3 \ No newline at end of file diff --git a/docker/compose/local-s3tests-compose.yml b/docker/compose/local-s3tests-compose.yml index 1db67e02b..9176c7bef 100644 --- a/docker/compose/local-s3tests-compose.yml +++ b/docker/compose/local-s3tests-compose.yml @@ -24,7 +24,7 @@ services: - 8888:8888 - 18888:18888 - 8000:8000 - command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000' + command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false' volumes: - ./s3.json:/etc/seaweedfs/s3.json depends_on: @@ -38,7 +38,7 @@ services: S3TEST_CONF: "s3tests.conf" NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" - NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" + NOSETESTS_EXCLUDE: "(get_bucket_encryption|delete_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)" depends_on: - master - volume diff --git a/docker/compose/local-sync-mount-compose.yml b/docker/compose/local-sync-mount-compose.yml new file mode 100644 index 000000000..fec866698 --- /dev/null +++ b/docker/compose/local-sync-mount-compose.yml @@ -0,0 +1,21 @@ +version: '3.9' +services: + node1: + image: chrislusf/seaweedfs:local + command: "server -master -volume -filer" + mount1: + image: chrislusf/seaweedfs:local + privileged: true + command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate" + node2: + image: chrislusf/seaweedfs:local + ports: + - 7888:8888 + command: "server -master -volume -filer" + mount2: + image: chrislusf/seaweedfs:local + privileged: true + command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate" + sync: + image: chrislusf/seaweedfs:local + command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug" diff --git a/docker/compose/master-cloud.toml b/docker/compose/master-cloud.toml index 17289c114..6ddb14e12 100644 --- a/docker/compose/master-cloud.toml +++ b/docker/compose/master-cloud.toml @@ -28,3 +28,4 @@ sleep_minutes = 17 # sleep minutes between each script execution region = "us-east-2" bucket = "volume_bucket" # an existing bucket endpoint = "http://server2:8333" + storage_class = "STANDARD_IA" diff --git a/docker/compose/test-etcd-filer.yml b/docker/compose/test-etcd-filer.yml new file mode 100644 index 000000000..400bd0fae --- /dev/null +++ b/docker/compose/test-etcd-filer.yml @@ -0,0 +1,61 @@ +version: '2' + +services: + etcd: + image: quay.io/coreos/etcd:v3.5.4 + command: "etcd --advertise-client-urls http://etcd:2379 --listen-client-urls http://0.0.0.0:2379" + ports: + - 2379:2379 + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=1024" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" + depends_on: + - master + s3: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + - 8333:8333 + command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333' + environment: + WEED_LEVELDB2_ENABLED: 'false' + WEED_ETCD_ENABLED: 'true' + WEED_ETCD_SERVERS: "http://etcd:2379" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + depends_on: + - etcd + - master + - volume + registry: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 diff --git a/docker/compose/test-ydb-filer.yml b/docker/compose/test-ydb-filer.yml new file mode 100644 index 000000000..c0c31fe5b --- /dev/null +++ b/docker/compose/test-ydb-filer.yml @@ -0,0 +1,35 @@ +version: '2' + +services: + ydb: + image: cr.yandex/yc/yandex-docker-local-ydb + ports: + - 2135:2135 + - 8765:8765 + - 2136:2136 + environment: + - YDB_DEFAULT_LOG_LEVEL=DEBUG + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + s3: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8888:8888 + - 8000:8000 + - 18888:18888 + command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + environment: + WEED_LEVELDB2_ENABLED: "false" + WEED_YDB_ENABLED: "true" + WEED_YDB_DSN: "grpc://ydb:2136/?database=local" + WEED_YDB_PREFIX: "seaweedfs" + YDB_ANONYMOUS_CREDENTIALS: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + depends_on: + - ydb \ No newline at end of file diff --git a/docker/compose/tls.env b/docker/compose/tls.env index a82954c4f..3a52fce52 100644 --- a/docker/compose/tls.env +++ b/docker/compose/tls.env @@ -11,4 +11,6 @@ WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" -WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" \ No newline at end of file +WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" +#GRPC_GO_LOG_SEVERITY_LEVEL=info +#GRPC_GO_LOG_VERBOSITY_LEVEL=2 \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a5a240575..80a7fe586 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -24,7 +24,7 @@ case "$1" in 'master') ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024" shift - exec /usr/bin/weed master $ARGS $@ + exec /usr/bin/weed -logtostderr=true master $ARGS $@ ;; 'volume') @@ -33,7 +33,7 @@ case "$1" in ARGS="-dir=/data" fi shift - exec /usr/bin/weed volume $ARGS $@ + exec /usr/bin/weed -logtostderr=true volume $ARGS $@ ;; 'server') @@ -42,31 +42,27 @@ case "$1" in ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024" fi shift - exec /usr/bin/weed server $ARGS $@ + exec /usr/bin/weed -logtostderr=true server $ARGS $@ ;; 'filer') ARGS="" shift - exec /usr/bin/weed filer $ARGS $@ + exec /usr/bin/weed -logtostderr=true filer $ARGS $@ ;; 's3') ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE" shift - exec /usr/bin/weed s3 $ARGS $@ + exec /usr/bin/weed -logtostderr=true s3 $ARGS $@ ;; - 'cronjob') - MASTER=${WEED_MASTER-localhost:9333} - FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *} - echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab - BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *} - echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab - echo "Running Crontab:" - cat /crontab - exec supercronic /crontab - ;; + 'shell') + ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS" + shift + exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS + ;; + *) exec /usr/bin/weed $@ ;; diff --git a/docker/filer_rocksdb.toml b/docker/filer_rocksdb.toml new file mode 100644 index 000000000..c1c74a64e --- /dev/null +++ b/docker/filer_rocksdb.toml @@ -0,0 +1,3 @@ +[rocksdb] +enabled = true +dir = "/data/filer_rocksdb" diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 34f669d56..2f0ead441 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -8,6 +8,7 @@ scrape_configs: static_configs: - targets: - 'prometheus:9090' + - 'master:9324' - 'volume:9325' - 'filer:9326' - 's3:9327' \ No newline at end of file diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml index f7d02a105..cce1c39ef 100644 --- a/docker/seaweedfs-compose.yml +++ b/docker/seaweedfs-compose.yml @@ -6,14 +6,15 @@ services: ports: - 9333:9333 - 19333:19333 - command: "master -ip=master" + - 9324:9324 + command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324" volume: image: chrislusf/seaweedfs # use a remote image ports: - 8080:8080 - 18080:18080 - 9325:9325 - command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325' + command: 'volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325' depends_on: - master filer: @@ -22,28 +23,18 @@ services: - 8888:8888 - 18888:18888 - 9326:9326 - command: 'filer -master="master:9333" -metricsPort=9326' + command: 'filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326' tty: true stdin_open: true depends_on: - master - volume - cronjob: - image: chrislusf/seaweedfs # use a remote image - command: 'cronjob' - environment: - # Run re-replication every 2 minutes - CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *' - WEED_MASTER: master:9333 # Default: localhost:9333 - depends_on: - - master - - volume s3: image: chrislusf/seaweedfs # use a remote image ports: - 8333:8333 - 9327:9327 - command: 's3 -filer="filer:8888" -metricsPort=9327' + command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0 -metricsPort=9327' depends_on: - master - volume diff --git a/docker/seaweedfs.sql b/docker/seaweedfs.sql index 38ebc575c..a27eb7081 100644 --- a/docker/seaweedfs.sql +++ b/docker/seaweedfs.sql @@ -1,6 +1,6 @@ CREATE DATABASE IF NOT EXISTS seaweedfs; CREATE USER IF NOT EXISTS 'seaweedfs'@'%' IDENTIFIED BY 'secret'; -GRANT ALL PRIVILEGES ON seaweedfs_fast.* TO 'seaweedfs'@'%'; +GRANT ALL PRIVILEGES ON seaweedfs.* TO 'seaweedfs'@'%'; FLUSH PRIVILEGES; USE seaweedfs; CREATE TABLE IF NOT EXISTS filemeta ( diff --git a/go.mod b/go.mod index 721f95065..cda1f1af8 100644 --- a/go.mod +++ b/go.mod @@ -1,27 +1,24 @@ module github.com/chrislusf/seaweedfs -go 1.17 +go 1.18 require ( - cloud.google.com/go v0.58.0 // indirect - cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.9.0 + cloud.google.com/go v0.102.0 // indirect + cloud.google.com/go/pubsub v1.22.2 + cloud.google.com/go/storage v1.22.1 github.com/Azure/azure-pipeline-go v0.2.3 - github.com/Azure/azure-storage-blob-go v0.14.0 - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect - github.com/OneOfOne/xxhash v1.2.2 - github.com/Shopify/sarama v1.23.1 - github.com/aws/aws-sdk-go v1.35.3 + github.com/Azure/azure-storage-blob-go v0.15.0 + github.com/OneOfOne/xxhash v1.2.8 + github.com/Shopify/sarama v1.34.1 + github.com/aws/aws-sdk-go v1.44.37 github.com/beorn7/perks v1.0.1 // indirect github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash v1.1.0 - github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/chrislusf/raft v1.0.7 - github.com/colinmarc/hdfs/v2 v2.2.0 + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/chrislusf/raft v1.0.9 github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.0.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/disintegration/imaging v1.6.2 @@ -34,91 +31,77 @@ require ( github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect - github.com/fclairamb/ftpserverlib v0.8.0 - github.com/frankban/quicktest v1.7.2 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/fclairamb/ftpserverlib v0.18.0 + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-errors/errors v1.1.1 // indirect - github.com/go-redis/redis/v8 v8.4.4 - github.com/go-sql-driver/mysql v1.5.0 - github.com/go-stack/stack v1.8.0 // indirect + github.com/go-redis/redis/v8 v8.11.5 + github.com/go-redsync/redsync/v4 v4.5.0 + github.com/go-sql-driver/mysql v1.6.0 + github.com/go-stack/stack v1.8.1 // indirect github.com/go-zookeeper/zk v1.0.2 // indirect github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d - github.com/golang-jwt/jwt v3.2.1+incompatible - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e - github.com/golang/protobuf v1.4.3 + github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.0.0 - github.com/google/go-cmp v0.5.5 // indirect - github.com/google/uuid v1.2.0 - github.com/google/wire v0.4.0 // indirect - github.com/googleapis/gax-go v2.0.2+incompatible // indirect - github.com/googleapis/gax-go/v2 v2.0.5 // indirect - github.com/gorilla/mux v1.7.4 - github.com/gorilla/websocket v1.4.1 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 + github.com/google/btree v1.1.2 + github.com/google/go-cmp v0.5.8 // indirect + github.com/google/uuid v1.3.0 + github.com/google/wire v0.5.0 // indirect + github.com/googleapis/gax-go/v2 v2.4.0 // indirect + github.com/gorilla/mux v1.8.0 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.1 - github.com/jinzhu/copier v0.2.8 + github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect + github.com/jinzhu/copier v0.3.5 github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.11 - github.com/jstemmer/go-junit-report v0.9.1 // indirect - github.com/karlseguin/ccache/v2 v2.0.7 + github.com/json-iterator/go v1.1.12 + github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.10.9 // indirect - github.com/klauspost/cpuid v1.2.1 // indirect - github.com/klauspost/reedsolomon v1.9.2 - github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/klauspost/compress v1.15.6 // indirect + github.com/klauspost/reedsolomon v1.9.16 github.com/kurin/blazer v0.5.3 - github.com/lib/pq v1.10.0 - github.com/magiconair/properties v1.8.1 // indirect - github.com/mailru/easyjson v0.7.1 // indirect - github.com/mattn/go-ieproxy v0.0.1 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect + github.com/lib/pq v1.10.6 + github.com/linxGnu/grocksdb v1.7.3 + github.com/magiconair/properties v1.8.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-ieproxy v0.0.3 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/nats-io/jwt v1.0.1 // indirect - github.com/nats-io/nats.go v1.10.0 // indirect - github.com/nats-io/nkeys v0.2.0 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/olivere/elastic/v7 v7.0.19 - github.com/pelletier/go-toml v1.7.0 // indirect - github.com/peterh/liner v1.1.0 - github.com/pierrec/lz4 v2.2.7+incompatible // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/olivere/elastic/v7 v7.0.32 + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/peterh/liner v1.2.2 github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.2.3 github.com/pquerna/cachecontrol v0.1.0 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/seaweedfs/fuse v1.2.0 - github.com/seaweedfs/goexif v1.0.2 - github.com/sirupsen/logrus v1.6.0 // indirect - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e + github.com/seaweedfs/goexif v2.0.0+incompatible + github.com/sirupsen/logrus v1.8.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.4.0 - github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 - github.com/stretchr/testify v1.7.0 + github.com/spf13/viper v1.12.0 + github.com/streadway/amqp v1.0.0 + github.com/stretchr/testify v1.7.3 + github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c - github.com/tidwall/gjson v1.8.1 - github.com/tidwall/match v1.0.3 - github.com/tidwall/pretty v1.1.0 // indirect - github.com/tikv/client-go v0.0.0-20210412055529-d811a08025fa - github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5 // indirect + github.com/tidwall/gjson v1.14.1 + github.com/tidwall/match v1.1.1 + github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 github.com/valyala/bytebufferpool v1.0.0 @@ -126,56 +109,129 @@ require ( github.com/viant/ptrie v0.3.0 github.com/viant/toolbox v0.33.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.0.2 // indirect - github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/xdg-go/scram v1.1.1 // indirect + github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - go.etcd.io/etcd v3.3.25+incompatible - go.mongodb.org/mongo-driver v1.7.0 - go.opencensus.io v0.22.4 // indirect - go.opentelemetry.io/otel v0.15.0 // indirect - gocloud.dev v0.20.0 - gocloud.dev/pubsub/natspubsub v0.20.0 - gocloud.dev/pubsub/rabbitpubsub v0.20.0 - golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect + go.etcd.io/etcd/client/v3 v3.5.4 + go.mongodb.org/mongo-driver v1.9.1 + go.opencensus.io v0.23.0 // indirect + gocloud.dev v0.25.0 + gocloud.dev/pubsub/natspubsub v0.25.0 + gocloud.dev/pubsub/rabbitpubsub v0.25.0 + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd golang.org/x/image v0.0.0-20200119044424-58c23975cae1 - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7 - golang.org/x/text v0.3.6 // indirect - golang.org/x/tools v0.1.4 - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/api v0.26.0 - google.golang.org/appengine v1.6.6 // indirect - google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 // indirect - google.golang.org/grpc v1.29.1 - google.golang.org/protobuf v1.26.0-rc.1 + golang.org/x/net v0.0.0-20220607020251-c690dde0001d + golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + google.golang.org/api v0.83.0 + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect + google.golang.org/grpc v1.47.0 + google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect - gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect - gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect - gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect - gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect modernc.org/b v1.0.0 // indirect - modernc.org/cc/v3 v3.33.5 // indirect - modernc.org/ccgo/v3 v3.9.4 // indirect - modernc.org/libc v1.9.5 // indirect - modernc.org/mathutil v1.2.2 // indirect - modernc.org/memory v1.0.4 // indirect + modernc.org/cc/v3 v3.36.0 // indirect + modernc.org/ccgo/v3 v3.16.6 // indirect + modernc.org/libc v1.16.7 // indirect + modernc.org/mathutil v1.4.1 // indirect + modernc.org/memory v1.1.1 // indirect modernc.org/opt v0.1.1 // indirect - modernc.org/sqlite v1.10.7 - modernc.org/strutil v1.1.0 // indirect + modernc.org/sqlite v1.17.2 + modernc.org/strutil v1.1.2 modernc.org/token v1.0.0 // indirect ) require ( + github.com/Jille/raft-grpc-transport v1.2.0 + github.com/arangodb/go-driver v1.3.2 + github.com/fluent/fluent-logger-golang v1.9.0 + github.com/hanwen/go-fuse/v2 v2.1.1-0.20220531082602-17a864ed5940 + github.com/hashicorp/raft v1.3.9 + github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 + github.com/tikv/client-go/v2 v2.0.1 + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 + github.com/ydb-platform/ydb-go-sdk/v3 v3.27.0 + google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 +) + +require ( + cloud.google.com/go/compute v1.6.1 // indirect + cloud.google.com/go/iam v0.3.0 // indirect + github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect + github.com/armon/go-metrics v0.3.10 // indirect + github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect + github.com/aws/smithy-go v1.11.2 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect + github.com/boltdb/bolt v1.3.1 // indirect + github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/fatih/color v1.13.0 // indirect + github.com/fclairamb/go-log v0.3.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect + github.com/hashicorp/go-hclog v1.2.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-msgpack v1.1.5 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/goidentity/v6 v6.0.1 // indirect - github.com/jcmturner/rpc/v2 v2.0.2 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.0.6 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-runewidth v0.0.7 // indirect + github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect + github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/philhofer/fwd v1.1.1 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect + github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect + github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.3.0 // indirect + github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect + github.com/tinylib/msgp v1.1.6 // indirect + github.com/twmb/murmur3 v1.1.3 // indirect + github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 // indirect + github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect + github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect + go.etcd.io/etcd/api/v3 v3.5.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.21.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect + gopkg.in/ini.v1 v1.66.4 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/uint128 v1.1.1 // indirect ) -// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse -// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft - -replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 +// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft diff --git a/go.sum b/go.sum index d8c825791..2eabdf880 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ -bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -12,223 +10,275 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.55.0/go.mod h1:ZHmoY+/lIMNkN2+fBmuTiqZ4inFhvQad8ft7MT8IV5Y= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.58.0 h1:vtAfVc723K3xKq1BQydk/FyCldnaNFhGhpJxaJzgRMQ= -cloud.google.com/go v0.58.0/go.mod h1:W+9FnSUw6nhVwXlFcp1eL+krq5+HQUJeUogSeJZZiWg= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.2.0/go.mod h1:iISCjWnTpnoJT1R287xRdjvQHJrxQOpeah4phb5D3h0= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw= +cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI= +cloud.google.com/go/kms v1.4.0 h1:iElbfoE61VeLhnZcGOltqL8HIly8Nhbe5t6JlH9GXjo= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= +cloud.google.com/go/monitoring v1.4.0/go.mod h1:y6xnxfwI3hTFWOdkOaD7nfJVlwuC3/mS/5kvtT131p4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.19.0/go.mod h1:/O9kmSe9bb9KRnIAWkzmqhPjHo6LtzGOBYd/kr06XSs= +cloud.google.com/go/pubsub v1.22.2 h1:e6A4rhtMX4opff/jDWApl4HwLtsCdV9VULVfpFRp6eo= +cloud.google.com/go/pubsub v1.22.2/go.mod h1:LBHGrtgM7+SGKCDKQu2pKIRtGwbZyJvRDkMk0594xdU= +cloud.google.com/go/secretmanager v1.3.0/go.mod h1:+oLTkouyiYiabAQNugCeTS3PAArGiMJuBqvJnJsyH+U= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.9.0 h1:oXnZyBjHB6hC8TnSle0AWW6pGJ29EuSo5ww+SFmdNBg= -cloud.google.com/go/storage v1.9.0/go.mod h1:m+/etGaqZbylxaNT876QGXqEHp4PR2Rq5GMqICWb9bU= -contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= -contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= -contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= +cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= +cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM= +contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= +contrib.go.opencensus.io/exporter/stackdriver v0.13.10/go.mod h1:I5htMbyta491eUxufwwZPQdcKvvgzMB4O9ni41YnIM8= +contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-amqp-common-go/v3 v3.2.1/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= +github.com/Azure/azure-amqp-common-go/v3 v3.2.2/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-service-bus-go v0.10.1/go.mod h1:E/FOceuKAFUfpbIJDKWz/May6guE+eGibfGT6q+n1to= -github.com/Azure/azure-storage-blob-go v0.9.0/go.mod h1:8UBPbiOhrMQ4pLPi3gA1tXnpjrS76UYE/fo5A40vf4g= -github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v59.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= +github.com/Azure/azure-service-bus-go v0.11.5/go.mod h1:MI6ge2CuQWBVq+ly456MY7XqNLJip5LO1iSFodbNLbU= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= -github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= -github.com/Azure/go-amqp v0.12.7/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/go-amqp v0.16.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= +github.com/Azure/go-amqp v0.16.4/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3 h1:OZEIaBbMdUE/Js+BQKlpO81XlISgipr6yDJ+PSwsgi4= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.22 h1:bXiQwDjrRmBQOE67bwlvUKAC1EU1yZTPQ38c+bstZws= +github.com/Azure/go-autorest/autorest v0.11.22/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.17 h1:esOPl2dhcz9P3jqBSJ8tPGEj2EqzPPT6zfyuloiogKY= +github.com/Azure/go-autorest/autorest/adal v0.9.17/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.9/go.mod h1:hg3/1yw0Bq87O3KvvnJoAh34/0zbP7SFizX/qN5JvjU= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= -github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/GoogleCloudPlatform/cloudsql-proxy v1.29.0/go.mod h1:spvB9eLJH9dutlbPSRmHvSXXHOwGRyeXh1jVdquA2G8= +github.com/Jille/raft-grpc-transport v1.2.0 h1:W/YSPz8IsirEyomjKmDog5Xk71o9+l4KhyMEX2TsgSs= +github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs= -github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= -github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/Shopify/sarama v1.34.1 h1:pVCQO7BMAK3s1jWhgi5v1W6lwZ6Veiekfc2vsgRS06Y= +github.com/Shopify/sarama v1.34.1/go.mod h1:NZSNswsnStpq8TUdFaqnpXm2Do6KRzTIjdBdVlL1YRM= +github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= +github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= -github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/arangodb/go-driver v1.3.2 h1:07cmMqPqEl+/MlosjFtVVakEbkPqBSUvw9S9/atX0+4= +github.com/arangodb/go-driver v1.3.2/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.31.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.34.30 h1:izATc/E0+HcT5YHmaQVjn7GHCoqaBxn0PGo6Zq5UNFA= -github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= -github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= +github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= +github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= +github.com/aws/aws-sdk-go-v2/config v1.15.3 h1:5AlQD0jhVXlGzwo+VORKiUuogkG7pQcLJNzIzK7eodw= +github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= +github.com/aws/aws-sdk-go-v2/credentials v1.11.2 h1:RQQ5fzclAKJyY5TvF+fkjJEwzK4hnxQCLOu5JXzDmQo= +github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 h1:LWPg5zjHV9oz/myQr4wMs0gi4CjnDN/ILmyZUFYXZsU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3/go.mod h1:0dHuD2HZZSiwfJSy1FO5bX1hQ1TxVV1QXXjpn3XUE44= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 h1:onz/VaaxZ7Z4V+WIN9Txly9XLTmoOh1oJ8XcAC3pako= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 h1:9stUQR/u2KXU6HkFJYlqnZEjBnbgrVbG6I5HN09xZh0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 h1:by9P+oy3P/CwggN4ClnW2D4oL91QV7pBzBICi1chZvQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3/go.mod h1:Seb8KNmD6kVTjwRjVEgOT5hPin6sq+v4C2ycJQDwuH8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 h1:Gh1Gpyh01Yvn7ilO/b/hr01WgNpaszfbKMUgqM186xQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3/go.mod h1:Bm/v2IaN6rZ+Op7zX+bOUMdL4fsrYZiD0dsjLhNKwZc= +github.com/aws/aws-sdk-go-v2/service/kms v1.16.3/go.mod h1:QuiHPBqlOFCi4LqdSskYYAWpQlx3PKmohy+rE2F+o5g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3/go.mod h1:g1qvDuRsJY+XghsV6zg00Z4KJ7DtFFCx8fJD2a491Ak= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU8rrre0/4a0pn2wgwiDvOEzoOjcJUBr67o= +github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 h1:7TdmoJJBwLFyakXjfrGztejwY5Ie1JEto7YFfznCmAw= +github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 h1:uHjK81fESbGy2Y9lspub1+C6VN5W2UXTDo2A/Pm4G0U= +github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM= +github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 h1:frW4ikGcxfAEDfmQqWgMLp+F1n4nRo9sF39OcIb5BkQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 h1:cJGRyzCSVwZC7zZZ1xbx9m32UnrKydRYhOvcD1NYP9Q= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= +github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= +github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg= github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= -github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chrislusf/raft v1.0.7 h1:reybAIwnQOTSgTj1YgflbJFWLSN0KVQSxe8gDZYa04o= -github.com/chrislusf/raft v1.0.7/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chrislusf/raft v1.0.9 h1:EGUpBUzQSzu7WG/jF16IeoySSuxyyK3lfoltcUckC3I= +github.com/chrislusf/raft v1.0.9/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM= -github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/parser v0.0.0-20160622100904-31edd927e5b1/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= -github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= @@ -240,32 +290,28 @@ github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/fclairamb/ftpserverlib v0.8.0 h1:ZsWUQ8Vg3Y8LIWRUAzVnFXY982Yztz2odDdK/UVJtik= -github.com/fclairamb/ftpserverlib v0.8.0/go.mod h1:xF4cy07oCHA9ZorKehsFGqA/1UHYaonmqHK2g3P1X8U= -github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fclairamb/ftpserverlib v0.18.0 h1:q/uz7jVFMoGEMswnA+nbaKEC5mzxXJOmhPE/Q3r7VZI= +github.com/fclairamb/ftpserverlib v0.18.0/go.mod h1:QhLRiCajhPG/2WwGgcsAqmlaYXX8KziNXtSe1BlRH+k= +github.com/fclairamb/go-log v0.3.0 h1:oSC7Zjt0FZIYC5xXahUUycKGkypSdr2srFPLsp7CLd0= +github.com/fclairamb/go-log v0.3.0/go.mod h1:XG61EiPlAXnPDN8SA4N3zeA+GyBJmVOCCo12WORx/gA= +github.com/fluent/fluent-logger-golang v1.9.0 h1:zUdY44CHX2oIUc7VTNZc+4m+ORuO/mldQDA7czhWXEg= +github.com/fluent/fluent-logger-golang v1.9.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= -github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.3/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -274,100 +320,69 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc= -github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY= -github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= +github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-redsync/redsync/v4 v4.5.0 h1:kJjDzn/iEbU+K/6w+O8b1rzuYIK/nP9EQRc5nXKW9x4= +github.com/go-redsync/redsync/v4 v4.5.0/go.mod h1:AfhgO1E6W3rlUTs6Zmz/B6qBZJFasV30lwo7nlizdDs= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d h1:k544nNVphXK4Yt0FTduvOvCfJabEY/DMkdNw0zpCwBE= github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -381,566 +396,502 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= -github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= -github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= -github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= +github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= +github.com/google/go-replayers/httpreplay v1.1.1 h1:H91sIMlt1NZzN7R+/ASswyouLJfW0WLW7fhyUFvDEkY= +github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE= -github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= +github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= -github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.11.0 h1:aT5ISUniaOTErogCQ+4pGoYNBB6rm6Fq3g1v8QwYGas= -github.com/grpc-ecosystem/grpc-gateway v1.11.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= -github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= +github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= +github.com/hanwen/go-fuse/v2 v2.1.1-0.20220531082602-17a864ed5940 h1:fZ4Oe3ArCbIe5Fzxr5yrDtAipC619dKuGYnrVlv5Hck= +github.com/hanwen/go-fuse/v2 v2.1.1-0.20220531082602-17a864ed5940/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.9 h1:9yuo1aR0bFTr1cw7pj3S2Bk6MhJCsnr2NAxvIBrP2x4= +github.com/hashicorp/raft v1.3.9/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc= +github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/hypnoglow/gormzap v0.3.0/go.mod h1:5Wom8B7Jl2oK0Im9hs6KQ+Kl92w4Y7gKCrj66rhyvw0= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.1 h1:IGSJfqBzMS6TA0oJ7DxXdyzPK563QHa8T2IqER2ggyQ= -github.com/jcmturner/gokrb5/v8 v8.4.1/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= -github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= -github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE= -github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= -github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM= -github.com/karlseguin/ccache/v2 v2.0.7/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= +github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= +github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.9 h1:pPRt1Z78crspaHISkpSSHjDlx+Tt9suHe519dsI0vF4= -github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18= -github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= +github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/reedsolomon v1.9.16 h1:mR0AwphBwqFv/I3B9AHtNKvzuowI1vrj8/3UX4XRmHA= +github.com/klauspost/reedsolomon v1.9.16/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk= github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= -github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linxGnu/grocksdb v1.7.3 h1:S9XiU4FviunvjNdNG+kWe2BoOy/2EKZSdDyeGmL0vDs= +github.com/linxGnu/grocksdb v1.7.3/go.mod h1:G4zrMNj2CP2aCXF61jbmZH81tu+kU3qU4rYpOU8WOL8= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-ieproxy v0.0.3 h1:YkaHmK1CzE5C4O7A3hv3TCbfNDPSCf0RKZFX+VhBeYk= +github.com/mattn/go-ieproxy v0.0.3/go.mod h1:6ZpRmhBaYuBX1U2za+9rC9iCGLsSp2tftelZne7CPko= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= -github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.2.6/go.mod h1:mQxQ0uHQ9FhEVPIcTSKwx2lqZEpXWWcCgA7R6NrWvvY= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/jwt v1.0.1 h1:71ivoESdfT2K/qDiw5YwX/3W9/dR7c+m83xiGOj/EZ4= -github.com/nats-io/jwt v1.0.1/go.mod h1:n3cvmLfBfnpV4JJRN7lRYCyZnw48ksGsbThGXEk4w9M= -github.com/nats-io/nats-server/v2 v2.0.0/go.mod h1:RyVdsHHvY4B6c9pWG+uRLpZ0h0XsqiuKp2XCTurP5LI= -github.com/nats-io/nats-server/v2 v2.1.2 h1:i2Ly0B+1+rzNZHHWtD4ZwKi+OU5l+uQo1iDHZ2PmiIc= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY= -github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= -github.com/nats-io/nkeys v0.2.0 h1:WXKF7diOaPU9cJdLD7nuzwasQy9vT1tBqzXZZf3AMJM= -github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY= +github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= +github.com/nats-io/nats-server/v2 v2.7.2 h1:+LEN8m0+jdCkiGc884WnDuxR+qj80/5arj+szKuRpRI= +github.com/nats-io/nats-server/v2 v2.7.2/go.mod h1:tckmrt0M6bVaDT3kmh9UrIq/CBOBBse+TpXQi5ldaa8= +github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d h1:GRSmEJutHkdoxKsRypP575IIdoXe7Bm6yHQF6GcDBnA= +github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w= -github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= -github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts= -github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= +github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= -github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= -github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.2.7+incompatible h1:Eerk9aiqeZo2QzsbWOAsELUf9ddvAxEdMY9LYze/DEc= -github.com/pierrec/lz4 v2.2.7+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20210104140916-41a0a3a87e75/go.mod h1:EONGys2gM5n14pII2vjmU/5VG3Dtj6kpqUT1GUZ4ysw= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= -github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= -github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 h1:LllgC9eGfqzkfubMgjKIDyZYaa609nNWAyNZtpy2B3M= -github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= -github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= -github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= -github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd h1:I8IeI8MNiZVKnwuXhcIIzz6pREcOSbq18Q31KYIzFVM= -github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd/go.mod h1:IVF+ijPSMZVtx2oIqxAg7ur6EyixtTYfOHwpfmlhqI4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210112051026-540f4cb76c1c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210219064844-c1844a4775d6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210219095907-b2375dcc80ad/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210531063847-f42e582bf0bb h1:6isHwZRl1fc9i1Mggiza2iQcfvVvYAAerFIs5t9msXg= -github.com/pingcap/kvproto v0.0.0-20210531063847-f42e582bf0bb/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 h1:4EUpHzPFHwleKkVALyMqQbQcNziPZvU+vhUT9Wzj93E= -github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 h1:c0d/sMTeftJQF9O5OHyezWwPrzf2FXcEE5HWwnq/Ahs= +github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8= -github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 h1:v7SipssMu4X1tVQOe3PIVE73keJNHCFXe4Cza5uNDZ8= -github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= -github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= -github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seaweedfs/fuse v1.2.0 h1:m4Ar3I0TY/Tzyb80lfJiXUKFcMBHqA5e/cZwxQtilZo= -github.com/seaweedfs/fuse v1.2.0/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594= -github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= -github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk= -github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw= -github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps= +github.com/seaweedfs/goexif v2.0.0+incompatible/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= -github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= +github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 h1:2MR0pKUzlP3SGgj5NYJe/zRYDwOu9ku6YHy+Iw7l5DM= -github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= -github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= -github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= -github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= -github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.3 h1:dAm0YRdRQlWojc3CrCRgPBzG5f941d0zvAKu7qY4e+I= +github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= +github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= +github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU= -github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= -github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= -github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/client-go v0.0.0-20210412055529-d811a08025fa h1:I7QMBJKfFua+5gGa1utGFKly9bpMl+bCma+sqhagxyc= -github.com/tikv/client-go v0.0.0-20210412055529-d811a08025fa/go.mod h1:r3S7aFm6itF36T5e4NL2Q60xHiJCMIswVYDySxkQPHM= -github.com/tikv/client-go/v2 v2.0.0-alpha h1:q4SCsd0ks03Y1+g0MR8WkrRFOMUIPFir6mS6xV5YJMk= -github.com/tikv/client-go/v2 v2.0.0-alpha/go.mod h1:CwWVYq/nNO7qMPzK6FY+PsXdkFlBfFYURkDRlKueD9U= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5 h1:7CJYiW8gKiI3IQOQSAZyqZq0GxB+bmrnZgk9QNZ1cPo= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5/go.mod h1:KwtZXt0JD+bP9bWW2ka0ir3Wp3oTEfZUTh22bs2sI4o= -github.com/tikv/pd v1.1.0-beta.0.20210122094357-c7aac753461a/go.mod h1:ws3WE0w9hXjOBX2dSmfc6dVvhWwQGTd+XZo+Jcb+yIc= -github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d h1:K0XnvsnT6ofLDuM8Rt3PuFQO4p8bNraeHYstspD316g= -github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= -github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= -github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tikv/client-go/v2 v2.0.1 h1:+K/VvVOxEOXKMtR83bs5Aj3lrYdTdTZdvH0apfAWW10= +github.com/tikv/client-go/v2 v2.0.1/go.mod h1:gaHSp8rnxZ0w36qb6QPPNPh9P0Mu5vAEwCQcc0Brni4= +github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 h1:7h/Oi4Zw6eGCeXh4Q4ZvKI4k7nBJVUq0c29YCcLwKPM= +github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8= +github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU= github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf1ed3Jm/Tb4830c/ZKDq+YoLsCt2rGQuT0= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ= @@ -949,115 +900,119 @@ github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY= github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec= github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE= github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= -github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= -github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 h1:S3bwscnat3pa188pvEuXDUstxPjCnGpn5bZJuRMkL+g= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= +github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= +github.com/ydb-platform/ydb-go-sdk/v3 v3.27.0 h1:paSdC12yRI19Vv9ej6qAjUu2/r/WaW/rzsUogd+lg34= +github.com/ydb-platform/ydb-go-sdk/v3 v3.27.0/go.mod h1:vXjmbeEAWlkVE5/ym3XHhtnWk7aDGGqFMKrfgwbRUkQ= +github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= +github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= +github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= +github.com/ydb-platform/ydb-go-yc-metadata v0.5.2/go.mod h1:82SQ4L3PewiEmFW4oTMc1sfPjODasIYxD/SKGsbK74s= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 h1:s71VGheLtWmCYsnNjf+s7XE8HsrZnd3EYGrLGWVm7nY= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547/go.mod h1:YoUyTScD3Vcv2RBm3eGVOq7i1ULiz3OuXoQFWOirmAM= -go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA= -go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= +go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c= +go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= -go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= -go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -gocloud.dev v0.20.0 h1:mbEKMfnyPV7W1Rj35R1xXfjszs9dXkwSOq2KoFr25g8= -gocloud.dev v0.20.0/go.mod h1:+Y/RpSXrJthIOM8uFNzWp6MRu9pFPNFEEZrQMxpkfIc= -gocloud.dev/pubsub/natspubsub v0.20.0 h1:DsOXYKfcSTh0SHKwuhpQAJmPLDj3+ARvYgBIupVPClE= -gocloud.dev/pubsub/natspubsub v0.20.0/go.mod h1:Zh7v7Q1DZjAoBwsavZLdvinMIO1eYE0PJTllMuX3VGA= -gocloud.dev/pubsub/rabbitpubsub v0.20.0 h1:hwupxLvWG8jTPNQ+9Q/YWZzyMagL9blTwWYYhoW4pco= -gocloud.dev/pubsub/rabbitpubsub v0.20.0/go.mod h1:xYCXmI3ixWuCW4s1KqyZpgKT90MMjdXdMlb0Kgmd7TM= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +gocloud.dev v0.25.0 h1:Y7vDq8xj7SyM848KXf32Krda2e6jQ4CLh/mTeCSqXtk= +gocloud.dev v0.25.0/go.mod h1:7HegHVCYZrMiU3IE1qtnzf/vRrDwLYnRNR3EhWX8x9Y= +gocloud.dev/pubsub/natspubsub v0.25.0 h1:k7saTEnbNZeJRWW2l46pri6f9Ha8ndoyvA1QNj92nt4= +gocloud.dev/pubsub/natspubsub v0.25.0/go.mod h1:Q8aqOBI3tmMg1IaxZBog6nq/fEyH9UNEtpO6X4aJH0I= +gocloud.dev/pubsub/rabbitpubsub v0.25.0 h1:jDAHvIH0h40quEuqusYXfK28sCABAMAnjLqLybu/aeo= +gocloud.dev/pubsub/rabbitpubsub v0.25.0/go.mod h1:gfOrMlNXnxzIYB3dK1mNenXeBwJjm2ZSRBgNzxan0/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1068,6 +1023,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd h1:zVFyTKZN/Q7mNRWSs1GOYnHM9NiFSJ54YVRsD0rNWT4= +golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1082,8 +1039,8 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1092,33 +1049,31 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1133,69 +1088,99 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220420153159-1850ba15e1be/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 h1:zwrSfklXn0gxyLRX/aR+q6cgHbV/ItVyzbPlbA+dkAw= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1205,34 +1190,70 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7 h1:lQ8Btl/sJr2+f4ql7ffKUKfnV0BsgsICvm0oEeINAQY= -golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1240,15 +1261,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= +golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1256,31 +1279,26 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1295,36 +1313,47 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200606014950-c42cb6316fb6/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200608174601-1b747fd94509/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201116002733-ac45abd4c88c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1337,24 +1366,54 @@ google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08= -google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= +google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.68.0/go.mod h1:sOM8pTpwgflXRhz+oC8H2Dr+UcbMqkPPWNJo88Q7TH8= +google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= +google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc= +google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1371,24 +1430,86 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200325114520-5b2d0af7952b/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 h1:i+Aiej6cta/Frzp13/swvwz5O00kYcSe0A/C5Wd7zX8= -google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220204002441-d6cc3cc0770e/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -1396,8 +1517,35 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= +google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= +google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE= +google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1/go.mod h1:PoKncN6QA5h/eFRzlCWpHSZnXF2pCtnBzAfeanB8OGQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1407,106 +1555,91 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/dutchcoders/goftp.v1 v1.0.0-20170301105846-ed59a591ce14 h1:tHqNpm9sPaE6BSuMLXBzgTwukQLdBEt4OYU2coQjEQQ= -gopkg.in/dutchcoders/goftp.v1 v1.0.0-20170301105846-ed59a591ce14/go.mod h1:nzmlZQ+UqB5+55CRTV/dOaiK8OrPl6Co96Ob8lH4Wxw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= +gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/oleiade/reflections.v1 v1.0.0/go.mod h1:SpA8pv+LUnF0FbB2hyRxc8XSng78D6iLBZ11PDb8Z5g= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.2.0/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= -modernc.org/cc/v3 v3.32.4/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= -modernc.org/cc/v3 v3.33.5 h1:gfsIOmcv80EelyQyOHn/Xhlzex8xunhQxWiJRMYmPrI= -modernc.org/cc/v3 v3.33.5/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= -modernc.org/ccgo/v3 v3.9.2/go.mod h1:gnJpy6NIVqkETT+L5zPsQFj7L2kkhfPMzOghRNv/CFo= -modernc.org/ccgo/v3 v3.9.4 h1:mt2+HyTZKxva27O6T4C9//0xiNQ/MornL3i8itM5cCs= -modernc.org/ccgo/v3 v3.9.4/go.mod h1:19XAY9uOrYnDhOgfHwCABasBvK69jgC4I8+rizbk3Bc= +modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= -modernc.org/libc v1.9.5 h1:zv111ldxmP7DJ5mOIqzRbza7ZDl3kh4ncKfASB2jIYY= -modernc.org/libc v1.9.5/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.7 h1:qzQtHhsZNpVPpeCu+aMIQldXeV1P0vRhSqCL0nOIJOA= +modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.0.4 h1:utMBrFcpnQDdNsmM6asmyH/FM9TqLPS7XF7otpJmrwM= -modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.10.7 h1:B4ITfAx3HxSxOOKZqKhw4vnrhM+kTY1HoJf2L7PQBCQ= -modernc.org/sqlite v1.10.7/go.mod h1:GXpJIZPNgRGqG0inyYDW18j9YpBpFUBn/weGI63hLLs= -modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= +modernc.org/sqlite v1.17.2 h1:TjmF36Wi5QcPYqRoAacV1cAyJ7xB/CD0ExpVUEMebnw= +modernc.org/sqlite v1.17.2/go.mod h1:GOQmuiXd6pTTes1Fi2s9apiCcD/wbKQtBZ0Nw6/etjM= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/tcl v1.5.2 h1:sYNjGr4zK6cDH74USl8wVJRrvDX6UOLpG0j4lFvR0W0= -modernc.org/tcl v1.5.2/go.mod h1:pmJYOLgpiys3oI4AeAafkcUfE+TKKilminxNyU/+Zlo= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.2 h1:iFBDH6j1Z0bN/Q9udJnnFoFpENA4252qe/7/5woE5MI= +modernc.org/strutil v1.1.2/go.mod h1:OYajnUAcI/MX+XD/Wx7v1bbdvcQSvxgtb0gC+u3d3eg= +modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= -modernc.org/z v1.0.1 h1:WyIDpEpAIx4Hel6q/Pcgj/VhaQV5XPJ2I6ryIYbjnpc= -modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/k8s/helm_charts2/Chart.yaml b/k8s/helm_charts2/Chart.yaml index b4caa24ee..e424a12ee 100644 --- a/k8s/helm_charts2/Chart.yaml +++ b/k8s/helm_charts2/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.65" -version: "2.65" +appVersion: "3.12" +version: "3.12" diff --git a/k8s/helm_charts2/templates/_helpers.tpl b/k8s/helm_charts2/templates/_helpers.tpl index a9ee89f03..688efaa23 100644 --- a/k8s/helm_charts2/templates/_helpers.tpl +++ b/k8s/helm_charts2/templates/_helpers.tpl @@ -113,21 +113,6 @@ Inject extra environment vars in the format key:value, if populated {{- end -}} {{- end -}} -{{/* Return the proper cronjob image */}} -{{- define "cronjob.image" -}} -{{- if .Values.cronjob.imageOverride -}} -{{- $imageOverride := .Values.cronjob.imageOverride -}} -{{- printf "%s" $imageOverride -}} -{{- else -}} -{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} -{{- $repositoryName := .Values.image.repository | toString -}} -{{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Chart.AppVersion | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} -{{- end -}} -{{- end -}} - - {{/* check if any PVC exists */}} {{- define "volume.pvc_exists" -}} {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}} diff --git a/k8s/helm_charts2/templates/cronjob.yaml b/k8s/helm_charts2/templates/cronjob.yaml deleted file mode 100644 index 4caf4bad1..000000000 --- a/k8s/helm_charts2/templates/cronjob.yaml +++ /dev/null @@ -1,58 +0,0 @@ -{{- if .Values.cronjob }} -{{- if .Values.cronjob.enabled }} -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: {{ include "seaweedfs.fullname" . }}-cronjob -spec: - schedule: "{{ .Values.cronjob.schedule }}" - startingDeadlineSeconds: 200 - concurrencyPolicy: Forbid - failedJobsHistoryLimit: 2 - successfulJobsHistoryLimit: 2 - jobTemplate: - spec: - backoffLimit: 2 - template: - spec: - {{- if .Values.cronjob.nodeSelector }} - nodeSelector: - {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }} - {{- end }} - {{- if .Values.cronjob.tolerations }} - tolerations: - {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }} - {{- end }} - restartPolicy: OnFailure - containers: - - name: shell - image: {{ template "cronjob.image" . }} - imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }} - resources: - {{- toYaml .Values.cronjob.resources| nindent 16 }} - command: - - sh - - -c - - | - set -ex - echo -e "lock\n\ - volume.balance -force \ - {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\ - {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\ - {{- if .Values.cronjob.enableFixReplication }} - volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\ - {{- end }} - unlock\n" | \ - /usr/bin/weed shell \ - {{- if .Values.cronjob.master }} - -master {{ .Values.cronjob.master }} \ - {{- else }} - -master {{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc:{{ .Values.master.port }} \ - {{- end }} - {{- if .Values.cronjob.filer }} - -filer {{ .Values.cronjob.filer }} - {{- else }} - -filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }} - {{- end }} -{{- end }} -{{- end }} diff --git a/k8s/helm_charts2/templates/filer-servicemonitor.yaml b/k8s/helm_charts2/templates/filer-servicemonitor.yaml index f07f6ebef..ed45442dc 100644 --- a/k8s/helm_charts2/templates/filer-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/filer-servicemonitor.yaml @@ -1,4 +1,5 @@ {{- if .Values.filer.metricsPort }} +{{- if .Values.global.monitoring.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -15,4 +16,5 @@ spec: selector: app: {{ template "seaweedfs.name" . }} component: filer +{{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/filer-statefulset.yaml b/k8s/helm_charts2/templates/filer-statefulset.yaml index fc1253479..94003819f 100644 --- a/k8s/helm_charts2/templates/filer-statefulset.yaml +++ b/k8s/helm_charts2/templates/filer-statefulset.yaml @@ -133,11 +133,6 @@ spec: -encryptVolumeData \ {{- end }} -ip=${POD_IP} \ - {{- if .Values.filer.enable_peers }} - {{- if gt (.Values.filer.replicas | int) 1 }} - -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' ) \ - {{- end }} - {{- end }} {{- if .Values.filer.s3.enabled }} -s3 \ -s3.port={{ .Values.filer.s3.port }} \ @@ -154,15 +149,21 @@ spec: {{- if .Values.filer.s3.enableAuth }} -s3.config=/etc/sw/seaweedfs_s3_config \ {{- end }} + {{- if .Values.filer.s3.auditLogConfig }} + -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \ + {{- end }} {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} - {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }} volumeMounts: - name: seaweedfs-filer-log-volume mountPath: "/logs/" - mountPath: /etc/sw name: config-users readOnly: true + {{- if .Values.filer.enablePVC }} + - name: data-filer + mountPath: /data + {{- end }} {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -185,7 +186,6 @@ spec: mountPath: /usr/local/share/ca-certificates/client/ {{- end }} {{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }} - {{- end }} ports: - containerPort: {{ .Values.filer.port }} name: swfs-filer @@ -252,16 +252,18 @@ spec: nodeSelector: {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }} {{- end }} -{{/* volumeClaimTemplates:*/}} -{{/* - metadata:*/}} -{{/* name: data-{{ .Release.Namespace }}*/}} -{{/* spec:*/}} -{{/* accessModes:*/}} -{{/* - ReadWriteOnce*/}} -{{/* resources:*/}} -{{/* requests:*/}} -{{/* storage: {{ .Values.filer.storage }}*/}} -{{/* {{- if .Values.filer.storageClass }}*/}} -{{/* storageClassName: {{ .Values.filer.storageClass }}*/}} -{{/* {{- end }}*/}} + {{- if .Values.filer.enablePVC }} + volumeClaimTemplates: + - metadata: + name: data-filer + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.filer.storage }} + {{- if .Values.filer.storageClass }} + storageClassName: {{ .Values.filer.storageClass }} + {{- end }} + {{- end }} {{- end }} diff --git a/k8s/helm_charts2/templates/ingress.yaml b/k8s/helm_charts2/templates/ingress.yaml index dcd52c138..f488ef67e 100644 --- a/k8s/helm_charts2/templates/ingress.yaml +++ b/k8s/helm_charts2/templates/ingress.yaml @@ -1,59 +1,67 @@ +{{- if .Values.filer.ingress.enabled }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1beta1 +{{- else }} apiVersion: extensions/v1beta1 +{{- end }} kind: Ingress metadata: - name: ingress-{{ template "seaweedfs.name" . }}-filer - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/auth-type: "basic" - nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" - nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer' - nginx.ingress.kubernetes.io/service-upstream: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$1 - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/enable-rewrite-log: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - nginx.ingress.kubernetes.io/configuration-snippet: | - sub_filter '' ' '; #add base url - sub_filter '="/' '="./'; #make absolute paths to relative - sub_filter '=/' '=./'; - sub_filter '/seaweedfsstatic' './seaweedfsstatic'; - sub_filter_once off; + name: ingress-{{ template "seaweedfs.name" . }}-filer + namespace: {{ .Release.Namespace }} + annotations: + {{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} spec: - rules: - - http: - paths: - - path: /sw-filer/?(.*) - backend: - serviceName: {{ template "seaweedfs.name" . }}-filer - servicePort: {{ .Values.filer.port }} + ingressClassName: {{ .Values.filer.ingress.className | quote }} + rules: + - http: + paths: + - path: /sw-filer/?(.*) + pathType: ImplementationSpecific + backend: +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: + name: {{ template "seaweedfs.name" . }}-filer + port: + number: {{ .Values.filer.port }} + #name: +{{- else }} + serviceName: {{ template "seaweedfs.name" . }}-filer + servicePort: {{ .Values.filer.port }} +{{- end }} +{{- end }} --- +{{- if .Values.master.ingress.enabled }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1beta1 +{{- else }} apiVersion: extensions/v1beta1 +{{- end }} kind: Ingress metadata: name: ingress-{{ template "seaweedfs.name" . }}-master + namespace: {{ .Release.Namespace }} annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/auth-type: "basic" - nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" - nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master' - nginx.ingress.kubernetes.io/service-upstream: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$1 - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/enable-rewrite-log: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - nginx.ingress.kubernetes.io/configuration-snippet: | - sub_filter '' ' '; #add base url - sub_filter '="/' '="./'; #make absolute paths to relative - sub_filter '=/' '=./'; - sub_filter '/seaweedfsstatic' './seaweedfsstatic'; - sub_filter_once off; + {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} spec: + ingressClassName: {{ .Values.master.ingress.className | quote }} rules: - http: paths: - path: /sw-master/?(.*) + pathType: ImplementationSpecific backend: +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: + name: {{ template "seaweedfs.name" . }}-master + port: + number: {{ .Values.master.port }} + #name: +{{- else }} serviceName: {{ template "seaweedfs.name" . }}-master servicePort: {{ .Values.master.port }} +{{- end }} +{{- end }} diff --git a/k8s/helm_charts2/templates/s3-deployment.yaml b/k8s/helm_charts2/templates/s3-deployment.yaml index b513e937b..aac09f328 100644 --- a/k8s/helm_charts2/templates/s3-deployment.yaml +++ b/k8s/helm_charts2/templates/s3-deployment.yaml @@ -10,7 +10,6 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} spec: - serviceName: {{ template "seaweedfs.name" . }}-s3 replicas: {{ .Values.s3.replicas }} selector: matchLabels: @@ -93,6 +92,9 @@ spec: {{- if .Values.s3.enableAuth }} -config=/etc/sw/seaweedfs_s3_config \ {{- end }} + {{- if .Values.s3.auditLogConfig }} + -auditLogConfig=/etc/sw/s3_auditLogConfig.json \ + {{- end }} -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }} volumeMounts: - name: logs @@ -127,7 +129,7 @@ spec: name: swfs-s3 readinessProbe: httpGet: - path: / + path: /status port: {{ .Values.s3.port }} scheme: HTTP initialDelaySeconds: 15 @@ -137,7 +139,7 @@ spec: timeoutSeconds: 10 livenessProbe: httpGet: - path: / + path: /status port: {{ .Values.s3.port }} scheme: HTTP initialDelaySeconds: 20 diff --git a/k8s/helm_charts2/templates/s3-servicemonitor.yaml b/k8s/helm_charts2/templates/s3-servicemonitor.yaml index 7f18f00f5..b549893c7 100644 --- a/k8s/helm_charts2/templates/s3-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/s3-servicemonitor.yaml @@ -1,4 +1,5 @@ {{- if .Values.s3.metricsPort }} +{{- if .Values.global.monitoring.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -15,4 +16,5 @@ spec: selector: app: {{ template "seaweedfs.name" . }} component: s3 +{{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/seaweedfs-s3-secret.yaml b/k8s/helm_charts2/templates/seaweedfs-s3-secret.yaml index 66fd5f28e..4e9189633 100644 --- a/k8s/helm_charts2/templates/seaweedfs-s3-secret.yaml +++ b/k8s/helm_charts2/templates/seaweedfs-s3-secret.yaml @@ -18,4 +18,12 @@ stringData: read_access_key_id: {{ $access_key_read }} read_secret_access_key: {{ $secret_key_read }} seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}' + {{- if .Values.filer.s3.auditLogConfig }} + filer_s3_auditLogConfig.json: | + {{ toJson .Values.filer.s3.auditLogConfig | nindent 4 }} + {{- end }} + {{- if .Values.s3.auditLogConfig }} + s3_auditLogConfig.json: | + {{ toJson .Values.s3.auditLogConfig | nindent 4 }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/service-account.yaml b/k8s/helm_charts2/templates/service-account.yaml index e82ef7d62..978452ca4 100644 --- a/k8s/helm_charts2/templates/service-account.yaml +++ b/k8s/helm_charts2/templates/service-account.yaml @@ -1,7 +1,7 @@ #hack for delete pod master after migration --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: seaweefds-rw-cr rules: @@ -16,7 +16,7 @@ metadata: namespace: {{ .Release.Namespace }} --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:serviceaccount:seaweefds-rw-sa:default subjects: @@ -26,4 +26,4 @@ subjects: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: seaweefds-rw-cr \ No newline at end of file + name: seaweefds-rw-cr diff --git a/k8s/helm_charts2/templates/volume-servicemonitor.yaml b/k8s/helm_charts2/templates/volume-servicemonitor.yaml index 1b286e9b6..90d70e8de 100644 --- a/k8s/helm_charts2/templates/volume-servicemonitor.yaml +++ b/k8s/helm_charts2/templates/volume-servicemonitor.yaml @@ -1,4 +1,5 @@ {{- if .Values.volume.metricsPort }} +{{- if .Values.global.monitoring.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -15,4 +16,5 @@ spec: selector: app: {{ template "seaweedfs.name" . }} component: volume +{{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/helm_charts2/templates/volume-statefulset.yaml b/k8s/helm_charts2/templates/volume-statefulset.yaml index 2168b2b80..de2703d14 100644 --- a/k8s/helm_charts2/templates/volume-statefulset.yaml +++ b/k8s/helm_charts2/templates/volume-statefulset.yaml @@ -40,7 +40,7 @@ spec: imagePullSecrets: - name: {{ .Values.global.imagePullSecrets }} {{- end }} - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 150 {{- if .Values.volume.priorityClassName }} priorityClassName: {{ .Values.volume.priorityClassName | quote }} {{- end }} diff --git a/k8s/helm_charts2/values.yaml b/k8s/helm_charts2/values.yaml index adc0f1d2c..b5d059ee9 100644 --- a/k8s/helm_charts2/values.yaml +++ b/k8s/helm_charts2/values.yaml @@ -41,8 +41,7 @@ master: grpcPort: 19333 ipBind: "0.0.0.0" volumePreallocate: false - #Master stops directing writes to oversized volumes - volumeSizeLimitMB: 30000 + volumeSizeLimitMB: 1000 loggingOverrideLevel: null #number of seconds between heartbeats, default 5 pulseSeconds: null @@ -109,6 +108,26 @@ master: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + ingress: + enabled: false + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/auth-type: "basic" + nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master' + nginx.ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/enable-rewrite-log: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/configuration-snippet: | + sub_filter '' ' '; #add base url + sub_filter '="/' '="./'; #make absolute paths to relative + sub_filter '=/' '=./'; + sub_filter '/seaweedfsstatic' './seaweedfsstatic'; + sub_filter_once off; + extraEnvironmentVars: WEED_MASTER_VOLUME_GROWTH_COPY_1: 7 WEED_MASTER_VOLUME_GROWTH_COPY_2: 6 @@ -161,7 +180,7 @@ volume: # Directories to store data files. dir[,dir]... (default "/tmp") dir: "/data" - # Directories to store index files. dir[,dir]... (default "/tmp") + # Directories to store index files. dir[,dir]... (default is the same as "dir") dir_idx: null # Maximum numbers of volumes, count[,count]... @@ -246,8 +265,6 @@ filer: maxMB: null # encrypt data on volume servers encryptVolumeData: false - # enable peers sync metadata, for leveldb (localdb for filer but with sync across) - enable_peers: false # Whether proxy or redirect to volume server during file GET request redirectOnRead: false @@ -255,12 +272,12 @@ filer: # Limit sub dir listing size (default 100000) dirListLimit: 100000 - # Turn off directory listing - disableDirListing: false - # Disable http request, only gRpc operations are allowed disableHttp: false + # enablePVC will create a pvc for filer for data persistence. + enablePVC: false + # storage and storageClass are the settings for configuring stateful # storage for the master pods. storage should be set to the disk size of # the attached volume. storageClass is the class of storage which defaults @@ -311,6 +328,26 @@ filer: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + ingress: + enabled: false + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/auth-type: "basic" + nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer' + nginx.ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/enable-rewrite-log: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/configuration-snippet: | + sub_filter '' ' '; #add base url + sub_filter '="/' '="./'; #make absolute paths to relative + sub_filter '=/' '=./'; + sub_filter '/seaweedfsstatic' './seaweedfsstatic'; + sub_filter_once off; + # extraEnvVars is a list of extra enviroment variables to set with the stateful set. extraEnvironmentVars: WEED_MYSQL_ENABLED: "true" @@ -323,6 +360,7 @@ filer: WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600" # enable usage of memsql as filer backend WEED_MYSQL_INTERPOLATEPARAMS: "true" + # if you want to use leveldb2, then should enable "enablePVC". or you may lose your data. WEED_LEVELDB2_ENABLED: "false" # with http DELETE, by default the filer would check whether a folder is empty. # recursive_delete will delete all sub folders and files, similar to "rm -Rf" @@ -340,6 +378,7 @@ filer: # enable user & permission to s3 (need to inject to all services) enableAuth: false skipAuthSecretCreation: false + auditLogConfig: {} s3: enabled: false @@ -356,6 +395,7 @@ s3: # enable user & permission to s3 (need to inject to all services) enableAuth: false skipAuthSecretCreation: false + auditLogConfig: {} # Suffix of the host name, {bucket}.{domainName} domainName: "" @@ -391,22 +431,6 @@ s3: size: "" storageClass: "" -cronjob: - enabled: true - master: "seaweedfs-master:9333" - filer: "seaweedfs-filer-client:8888" - tolerations: "" - nodeSelector: | - sw-backend: "true" - replication: - enable: true - collectionPattern: "" - schedule: "*/7 * * * *" - resources: null - # balance all volumes among volume servers - # ALL|EACH_COLLECTION| - collection: "" - certificates: commonName: "SeaweedFS CA" diff --git a/note/SeaweedFS_Gateway_RemoteObjectStore.png b/note/SeaweedFS_Gateway_RemoteObjectStore.png new file mode 100644 index 000000000..82c90e369 Binary files /dev/null and b/note/SeaweedFS_Gateway_RemoteObjectStore.png differ diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 70c5dbd31..33596f2a8 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.7 + 2.85 org.sonatype.oss @@ -14,7 +14,7 @@ - 3.9.1 + 3.16.1 1.23.0 30.0-jre @@ -60,7 +60,7 @@ org.apache.httpcomponents httpmime - 4.5.6 + 4.5.13 junit @@ -135,7 +135,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index 82cf5e82b..865467cdc 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.7 + 2.85 org.sonatype.oss @@ -60,7 +60,7 @@ org.apache.httpcomponents httpmime - 4.5.6 + 4.5.13 junit @@ -130,7 +130,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index c72c81ab7..29c8c459d 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.6.7 + 2.85 org.sonatype.oss @@ -60,7 +60,7 @@ org.apache.httpcomponents httpmime - 4.5.6 + 4.5.13 junit diff --git a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java index 58870d742..84b2beeb1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java @@ -15,7 +15,7 @@ public class ChunkCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterWrite(1, TimeUnit.HOURS) .build(); } diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index e70f6befa..10d263968 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -4,7 +4,6 @@ import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -15,7 +14,11 @@ public class FilerClient extends FilerGrpcClient { private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class); public FilerClient(String host, int grpcPort) { - super(host, grpcPort); + super(host, grpcPort-10000, grpcPort); + } + + public FilerClient(String host, int port, int grpcPort) { + super(host, port, grpcPort); } public static String toFileId(FilerProto.FileId fid) { @@ -108,9 +111,9 @@ public class FilerClient extends FilerGrpcClient { if ("/".equals(path)) { return true; } - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; mkdirs(parent, mode, uid, gid, userName, groupNames); @@ -129,35 +132,32 @@ public class FilerClient extends FilerGrpcClient { public boolean mv(String oldPath, String newPath) { - File oldPathFile = new File(oldPath); - String oldParent = oldPathFile.getParent().replace('\\','/'); - String oldName = oldPathFile.getName(); + String[] oldDirAndName = SeaweedUtil.toDirAndName(oldPath); + String oldParent = oldDirAndName[0]; + String oldName = oldDirAndName[1]; - File newPathFile = new File(newPath); - String newParent = newPathFile.getParent().replace('\\','/'); - String newName = newPathFile.getName(); + String[] newDirAndName = SeaweedUtil.toDirAndName(newPath); + String newParent = newDirAndName[0]; + String newName = newDirAndName[1]; return atomicRenameEntry(oldParent, oldName, newParent, newName); } public boolean exists(String path){ - File pathFile = new File(path); - String parent = pathFile.getParent(); - String entryName = pathFile.getName(); - if(parent == null) { - parent = path; - entryName =""; - } - return lookupEntry(parent, entryName) != null; + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String entryName = dirAndName[1]; + + return lookupEntry(parent, entryName) != null; } public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; return deleteEntry( parent, @@ -168,17 +168,19 @@ public class FilerClient extends FilerGrpcClient { } public boolean touch(String path, int mode) { + String currentUser = System.getProperty("user.name"); long now = System.currentTimeMillis() / 1000L; return touch(path, now, mode, 0, 0, currentUser, new String[]{}); + } public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; FilerProto.Entry entry = lookupEntry(parent, name); if (entry == null) { @@ -366,6 +368,7 @@ public class FilerClient extends FilerGrpcClient { .setPathPrefix(prefix) .setClientName(clientName) .setSinceNs(sinceNs) + .setClientId(this.randomClientId) .build() ); } diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 6c57e2e0d..0a2e6332e 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; import java.util.HashMap; import java.util.Map; +import java.util.Random; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { @@ -30,6 +31,7 @@ public class FilerGrpcClient { public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1; public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2; public final Map vidLocations = new HashMap<>(); + protected int randomClientId; private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; @@ -40,11 +42,11 @@ public class FilerGrpcClient { private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; private String filerAddress; - public FilerGrpcClient(String host, int grpcPort) { - this(host, grpcPort, sslContext); + public FilerGrpcClient(String host, int port, int grpcPort) { + this(host, port, grpcPort, sslContext); } - public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) { + public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) { this(sslContext == null ? ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() @@ -54,7 +56,7 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); - filerAddress = String.format("%s:%d", host, grpcPort - 10000); + filerAddress = SeaweedUtil.joinHostPort(host, port); FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( @@ -62,6 +64,7 @@ public class FilerGrpcClient { cipher = filerConfigurationResponse.getCipher(); collection = filerConfigurationResponse.getCollection(); replication = filerConfigurationResponse.getReplication(); + randomClientId = new Random().nextInt(); } diff --git a/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java b/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java new file mode 100644 index 000000000..2eba4f808 --- /dev/null +++ b/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java @@ -0,0 +1,109 @@ +package seaweedfs.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class ReadChunks { + + public static List readResolvedChunks(List chunkList) throws IOException { + List points = new ArrayList<>(chunkList.size() * 2); + for (FilerProto.FileChunk chunk : chunkList) { + points.add(new Point(chunk.getOffset(), chunk, true)); + points.add(new Point(chunk.getOffset() + chunk.getSize(), chunk, false)); + } + Collections.sort(points, new Comparator() { + @Override + public int compare(Point a, Point b) { + int x = (int) (a.x - b.x); + if (a.x != b.x) { + return (int) (a.x - b.x); + } + if (a.ts != b.ts) { + return (int) (a.ts - b.ts); + } + if (!a.isStart) { + return -1; + } + return 1; + } + }); + + long prevX = 0; + List visibles = new ArrayList<>(); + ArrayList queue = new ArrayList<>(); + for (Point point : points) { + if (point.isStart) { + if (queue.size() > 0) { + int lastIndex = queue.size() - 1; + Point lastPoint = queue.get(lastIndex); + if (point.x != prevX && lastPoint.ts < point.ts) { + addToVisibles(visibles, prevX, lastPoint, point); + prevX = point.x; + } + } + // insert into queue + for (int i = queue.size(); i >= 0; i--) { + if (i == 0 || queue.get(i - 1).ts <= point.ts) { + if (i == queue.size()) { + prevX = point.x; + } + queue.add(i, point); + break; + } + } + } else { + int lastIndex = queue.size() - 1; + int index = lastIndex; + Point startPoint = null; + for (; index >= 0; index--) { + startPoint = queue.get(index); + if (startPoint.ts == point.ts) { + queue.remove(index); + break; + } + } + if (index == lastIndex && startPoint != null) { + addToVisibles(visibles, prevX, startPoint, point); + prevX = point.x; + } + } + } + + return visibles; + + } + + private static void addToVisibles(List visibles, long prevX, Point startPoint, Point point) { + if (prevX < point.x) { + FilerProto.FileChunk chunk = startPoint.chunk; + visibles.add(new SeaweedRead.VisibleInterval( + prevX, + point.x, + chunk.getFileId(), + chunk.getMtime(), + prevX - chunk.getOffset(), + chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x, + chunk.getCipherKey().toByteArray(), + chunk.getIsCompressed() + )); + } + } + + static class Point { + long x; + long ts; + FilerProto.FileChunk chunk; + boolean isStart; + + public Point(long x, FilerProto.FileChunk chunk, boolean isStart) { + this.x = x; + this.ts = chunk.getMtime(); + this.chunk = chunk; + this.isStart = isStart; + } + } + +} diff --git a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java index 39c17644b..0d912272b 100644 --- a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java @@ -14,10 +14,10 @@ public class RemoteUtil { String dir = SeaweedOutputStream.getParentDirectory(fullpath); String name = SeaweedOutputStream.getFileName(fullpath); - final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub() - .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder() + final FilerProto.CacheRemoteObjectToLocalClusterResponse response = filerClient.getBlockingStub() + .cacheRemoteObjectToLocalCluster(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder() .setDirectory(dir).setName(name).build()); - return downloadToLocalResponse.getEntry(); + return response.getEntry(); } } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java index 8d0ebd755..979decb8d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java @@ -36,7 +36,7 @@ public class SeaweedCipher { byte[] encryptedText = AES_cipherInstance.doFinal(clearTextbytes, offset, length); byte[] iv = AES_cipherInstance.getIV(); - byte[] message = new byte[GCM_NONCE_LENGTH + clearTextbytes.length + GCM_TAG_LENGTH]; + byte[] message = new byte[GCM_NONCE_LENGTH + length + GCM_TAG_LENGTH]; System.arraycopy(iv, 0, message, 0, GCM_NONCE_LENGTH); System.arraycopy(encryptedText, 0, message, GCM_NONCE_LENGTH, encryptedText.length); diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 011462a17..41033befb 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -226,96 +226,8 @@ public class SeaweedRead { chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList); - FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]); - Arrays.sort(chunks, new Comparator() { - @Override - public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) { - // if just a.getMtime() - b.getMtime(), it will overflow! - if (a.getMtime() < b.getMtime()) { - return -1; - } else if (a.getMtime() > b.getMtime()) { - return 1; - } - return 0; - } - }); - - List visibles = new ArrayList<>(); - for (FilerProto.FileChunk chunk : chunks) { - List newVisibles = new ArrayList<>(); - visibles = mergeIntoVisibles(visibles, newVisibles, chunk); - } - - return visibles; - } - - private static List mergeIntoVisibles(List visibles, - List newVisibles, - FilerProto.FileChunk chunk) { - VisibleInterval newV = new VisibleInterval( - chunk.getOffset(), - chunk.getOffset() + chunk.getSize(), - chunk.getFileId(), - chunk.getMtime(), - 0, - true, - chunk.getCipherKey().toByteArray(), - chunk.getIsCompressed() - ); - - // easy cases to speed up - if (visibles.size() == 0) { - visibles.add(newV); - return visibles; - } - if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) { - visibles.add(newV); - return visibles; - } - - for (VisibleInterval v : visibles) { - if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) { - newVisibles.add(new VisibleInterval( - v.start, - chunk.getOffset(), - v.fileId, - v.modifiedTime, - v.chunkOffset, - false, - v.cipherKey, - v.isCompressed - )); - } - long chunkStop = chunk.getOffset() + chunk.getSize(); - if (v.start < chunkStop && chunkStop < v.stop) { - newVisibles.add(new VisibleInterval( - chunkStop, - v.stop, - v.fileId, - v.modifiedTime, - v.chunkOffset + (chunkStop - v.start), - false, - v.cipherKey, - v.isCompressed - )); - } - if (chunkStop <= v.start || v.stop <= chunk.getOffset()) { - newVisibles.add(v); - } - } - newVisibles.add(newV); - - // keep everything sorted - for (int i = newVisibles.size() - 1; i >= 0; i--) { - if (i > 0 && newV.start < newVisibles.get(i - 1).start) { - newVisibles.set(i, newVisibles.get(i - 1)); - } else { - newVisibles.set(i, newV); - break; - } - } + return ReadChunks.readResolvedChunks(chunkList); - return newVisibles; } public static String parseVolumeId(String fileId) { diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index c465d935f..027e49b96 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -27,4 +27,30 @@ public class SeaweedUtil { public static CloseableHttpClient getClosableHttpClient() { return httpClient; } + + public static String[] toDirAndName(String fullpath) { + if (fullpath == null) { + return new String[]{"/", ""}; + } + if (fullpath.endsWith("/")) { + fullpath = fullpath.substring(0, fullpath.length() - 1); + } + if (fullpath.length() == 0) { + return new String[]{"/", ""}; + } + int sep = fullpath.lastIndexOf("/"); + String parent = sep == 0 ? "/" : fullpath.substring(0, sep); + String name = fullpath.substring(sep + 1); + return new String[]{parent, name}; + } + + public static String joinHostPort(String host, int port) { + if (host.startsWith("[") && host.endsWith("]")) { + return host + ":" + port; + } + if (host.indexOf(':')>=0) { + return "[" + host + "]:" + port; + } + return host + ":" + port; + } } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 3f9d79b99..1ee745ed0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -29,11 +29,31 @@ public class SeaweedWrite { final byte[] bytes, final long bytesOffset, final long bytesLength, final String path) throws IOException { - FilerProto.FileChunk.Builder chunkBuilder = writeChunk( - replication, filerClient, offset, bytes, bytesOffset, bytesLength, path); - synchronized (entry) { - entry.addChunks(chunkBuilder); + + IOException lastException = null; + for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { + try { + FilerProto.FileChunk.Builder chunkBuilder = writeChunk( + replication, filerClient, offset, bytes, bytesOffset, bytesLength, path); + lastException = null; + synchronized (entry) { + entry.addChunks(chunkBuilder); + } + break; + } catch (IOException ioe) { + LOG.debug("writeData:{}", ioe); + lastException = ioe; + } + try { + Thread.sleep(waitTime); + } catch (InterruptedException e) { + } + } + + if (lastException != null) { + throw lastException; } + } public static FilerProto.FileChunk.Builder writeChunk(final String replication, @@ -59,7 +79,7 @@ public class SeaweedWrite { String fileId = response.getFileId(); String auth = response.getAuth(); - String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl()); + String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl()); ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY; byte[] cipherKey = null; diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index 86263fff9..6f3d9d8c1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -15,7 +15,7 @@ public class VolumeIdCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(5, TimeUnit.MINUTES) + .expireAfterWrite(5, TimeUnit.MINUTES) .build(); } diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index bb461936c..bd0932cb8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -30,6 +30,8 @@ service SeaweedFiler { rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) { } + rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) { + } rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { } @@ -46,6 +48,9 @@ service SeaweedFiler { rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { } + rpc Ping (PingRequest) returns (PingResponse) { + } + rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) { } @@ -67,7 +72,7 @@ service SeaweedFiler { rpc KvPut (KvPutRequest) returns (KvPutResponse) { } - rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) { + rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) { } } @@ -112,6 +117,7 @@ message Entry { bytes content = 9; // if not empty, the file content RemoteEntry remote_entry = 10; + int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled. } message FullEntry { @@ -160,14 +166,13 @@ message FuseAttributes { uint32 gid = 5; int64 crtime = 6; // unix time in seconds string mime = 7; - string replication = 8; - string collection = 9; int32 ttl_sec = 10; string user_name = 11; // for hdfs repeated string group_name = 12; // for hdfs string symlink_target = 13; bytes md5 = 14; - string disk_type = 15; + uint32 rdev = 16; + uint64 inode = 17; } message CreateEntryRequest { @@ -176,6 +181,7 @@ message CreateEntryRequest { bool o_excl = 3; bool is_from_other_cluster = 4; repeated int32 signatures = 5; + bool skip_check_parent_directory = 6; } message CreateEntryResponse { @@ -225,6 +231,18 @@ message AtomicRenameEntryRequest { message AtomicRenameEntryResponse { } +message StreamRenameEntryRequest { + string old_directory = 1; + string old_name = 2; + string new_directory = 3; + string new_name = 4; + repeated int32 signatures = 5; +} +message StreamRenameEntryResponse { + string directory = 1; + EventNotification event_notification = 2; + int64 ts_ns = 3; +} message AssignVolumeRequest { int32 count = 1; string collection = 2; @@ -233,18 +251,18 @@ message AssignVolumeRequest { string data_center = 5; string path = 6; string rack = 7; + string data_node = 9; string disk_type = 8; } message AssignVolumeResponse { string file_id = 1; - string url = 2; - string public_url = 3; int32 count = 4; string auth = 5; string collection = 6; string replication = 7; string error = 8; + Location location = 9; } message LookupVolumeRequest { @@ -258,6 +276,7 @@ message Locations { message Location { string url = 1; string public_url = 2; + uint32 grpc_port = 3; } message LookupVolumeResponse { map locations_map = 1; @@ -292,6 +311,16 @@ message StatisticsResponse { uint64 file_count = 6; } +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +} + message GetFilerConfigurationRequest { } message GetFilerConfigurationResponse { @@ -306,6 +335,7 @@ message GetFilerConfigurationResponse { int32 metrics_interval_sec = 10; string version = 11; string cluster_id = 12; + string filer_group = 13; } message SubscribeMetadataRequest { @@ -313,6 +343,9 @@ message SubscribeMetadataRequest { string path_prefix = 2; int64 since_ns = 3; int32 signature = 4; + repeated string path_prefixes = 6; + int32 client_id = 7; + int64 until_ns = 8; } message SubscribeMetadataResponse { string directory = 1; @@ -381,6 +414,9 @@ message FilerConf { bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; + string data_center = 9; + string rack = 10; + string data_node = 11; } repeated PathConf locations = 2; } @@ -388,10 +424,10 @@ message FilerConf { ///////////////////////// // Remote Storage related ///////////////////////// -message DownloadToLocalRequest { +message CacheRemoteObjectToLocalClusterRequest { string directory = 1; string name = 2; } -message DownloadToLocalResponse { +message CacheRemoteObjectToLocalClusterResponse { Entry entry = 1; } diff --git a/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java b/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java index 44b833c90..6ad9edb2c 100644 --- a/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java +++ b/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Random; public class SeaweedReadTest { @@ -13,17 +14,17 @@ public class SeaweedReadTest { public void testNonOverlappingVisibleIntervals() throws IOException { List chunks = new ArrayList<>(); chunks.add(FilerProto.FileChunk.newBuilder() - .setFileId("aaa") - .setOffset(0) - .setSize(100) - .setMtime(1000) - .build()); + .setFileId("aaa") + .setOffset(0) + .setSize(100) + .setMtime(1000) + .build()); chunks.add(FilerProto.FileChunk.newBuilder() - .setFileId("bbb") - .setOffset(100) - .setSize(133) - .setMtime(2000) - .build()); + .setFileId("bbb") + .setOffset(100) + .setSize(133) + .setMtime(2000) + .build()); List visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks); for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { @@ -61,4 +62,106 @@ public class SeaweedReadTest { } + + @Test + public void testReadResolvedChunks() throws IOException { + List chunks = new ArrayList<>(); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("a") + .setOffset(0) + .setSize(100) + .setMtime(1) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("b") + .setOffset(50) + .setSize(100) + .setMtime(2) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("c") + .setOffset(200) + .setSize(50) + .setMtime(3) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("d") + .setOffset(250) + .setSize(50) + .setMtime(4) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("e") + .setOffset(175) + .setSize(100) + .setMtime(5) + .build()); + + List visibleIntervals = ReadChunks.readResolvedChunks(chunks); + for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { + System.out.println("visible:" + visibleInterval); + } + + Assert.assertEquals(4, visibleIntervals.size()); + + SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0); + Assert.assertEquals(visibleInterval.start, 0); + Assert.assertEquals(visibleInterval.stop, 50); + Assert.assertEquals(visibleInterval.modifiedTime, 1); + Assert.assertEquals(visibleInterval.fileId, "a"); + + visibleInterval = visibleIntervals.get(1); + Assert.assertEquals(visibleInterval.start, 50); + Assert.assertEquals(visibleInterval.stop, 150); + Assert.assertEquals(visibleInterval.modifiedTime, 2); + Assert.assertEquals(visibleInterval.fileId, "b"); + + visibleInterval = visibleIntervals.get(2); + Assert.assertEquals(visibleInterval.start, 175); + Assert.assertEquals(visibleInterval.stop, 275); + Assert.assertEquals(visibleInterval.modifiedTime, 5); + Assert.assertEquals(visibleInterval.fileId, "e"); + + visibleInterval = visibleIntervals.get(3); + Assert.assertEquals(visibleInterval.start, 275); + Assert.assertEquals(visibleInterval.stop, 300); + Assert.assertEquals(visibleInterval.modifiedTime, 4); + Assert.assertEquals(visibleInterval.fileId, "d"); + + } + + + @Test + public void testRandomizedReadResolvedChunks() throws IOException { + Random random = new Random(); + int limit = 1024*1024; + long[] array = new long[limit]; + List chunks = new ArrayList<>(); + for (long ts=0;ts<1024;ts++){ + int x = random.nextInt(limit); + int y = random.nextInt(limit); + int size = Math.min(Math.abs(x-y), 1024); + chunks.add(randomWrite(array, Math.min(x,y), size, ts)); + } + + List visibleIntervals = ReadChunks.readResolvedChunks(chunks); + for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { + System.out.println("visible:" + visibleInterval); + for (int i = (int) visibleInterval.start; i com.github.chrislusf seaweedfs-client - 1.6.7 + 2.85 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.6.7 + 2.85 compile org.apache.hadoop hadoop-common - 2.9.2 + 2.10.1 compile - \ No newline at end of file + diff --git a/other/java/hdfs-over-ftp/pom.xml b/other/java/hdfs-over-ftp/pom.xml index 0db422db5..8b4f0e612 100644 --- a/other/java/hdfs-over-ftp/pom.xml +++ b/other/java/hdfs-over-ftp/pom.xml @@ -36,7 +36,7 @@ org.apache.hadoop hadoop-common - 3.2.1 + 3.2.3 org.apache.hadoop diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index bd31637ce..74f8e6240 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -86,7 +86,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh @@ -301,7 +301,7 @@ - 1.6.7 + 2.85 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index f15d24faa..eccbb54bf 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,8 +5,8 @@ 4.0.0 - 1.6.7 - 2.9.2 + 2.85 + 2.10.1 com.github.chrislusf @@ -105,7 +105,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 25395db7a..b6ea4c3bb 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host"; public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; + public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; @@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem { // get host information from uri (overrides info in conf) String host = uri.getHost(); host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host; - if (host == null) { - throw new IOException("Invalid host specified"); - } conf.set(FS_SEAWEED_FILER_HOST, host); // get port information from uri, (overrides info in conf) @@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem { port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port; conf.setInt(FS_SEAWEED_FILER_PORT, port); + int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000); + setConf(conf); this.uri = uri; - seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf); + seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf); } diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index f4e8c9349..2ba8e1a10 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -27,9 +27,8 @@ public class SeaweedFileSystemStore { private FilerClient filerClient; private Configuration conf; - public SeaweedFileSystemStore(String host, int port, Configuration conf) { - int grpcPort = 10000 + port; - filerClient = new FilerClient(host, grpcPort); + public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) { + filerClient = new FilerClient(host, port, grpcPort); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 4640b5a84..bbfd48ab9 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -86,7 +86,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh @@ -309,7 +309,7 @@ - 1.6.7 + 2.85 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index efcc1e4c0..f25ecf986 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,8 +5,8 @@ 4.0.0 - 1.6.7 - 3.1.1 + 2.85 + 3.2.3 com.github.chrislusf @@ -105,7 +105,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 25395db7a..b6ea4c3bb 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host"; public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; + public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; @@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem { // get host information from uri (overrides info in conf) String host = uri.getHost(); host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host; - if (host == null) { - throw new IOException("Invalid host specified"); - } conf.set(FS_SEAWEED_FILER_HOST, host); // get port information from uri, (overrides info in conf) @@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem { port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port; conf.setInt(FS_SEAWEED_FILER_PORT, port); + int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000); + setConf(conf); this.uri = uri; - seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf); + seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf); } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index f4e8c9349..2ba8e1a10 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -27,9 +27,8 @@ public class SeaweedFileSystemStore { private FilerClient filerClient; private Configuration conf; - public SeaweedFileSystemStore(String host, int port, Configuration conf) { - int grpcPort = 10000 + port; - filerClient = new FilerClient(host, grpcPort); + public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) { + filerClient = new FilerClient(host, port, grpcPort); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 3b9b222b4..88844b3c3 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -539,11 +539,12 @@ "step": 60 }, { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -645,11 +646,12 @@ "step": 60 }, { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -751,11 +753,11 @@ "step": 60 }, { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -864,7 +866,7 @@ "expr": "rate(SeaweedFS_s3_request_total[1m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "A", "step": 30 } diff --git a/other/metrics/grafana_seaweedfs_heartbeat.json b/other/metrics/grafana_seaweedfs_heartbeat.json new file mode 100644 index 000000000..9a52624cf --- /dev/null +++ b/other/metrics/grafana_seaweedfs_heartbeat.json @@ -0,0 +1,1932 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS-DEV", + "label": "prometheus-dev", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.6.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "${DS_PROMETHEUS-DEV}", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 10423, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": true, + "height": 251, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Request Duration 90th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 49, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Request Duration 95th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Request Duration 99th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 12, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(SeaweedFS_filer_request_total[1m]) * 5", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer QPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Filer", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Request Duration 90th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 57, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Request Duration 95th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 Request Duration 99th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 12, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(SeaweedFS_s3_request_total[1m]) * 5", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "S3 API QPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "hideTimeOverride": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 12, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "A", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "All PUT, COPY, POST, LIST", + "refId": "C", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "GET and all other", + "refId": "B" + }, + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": "1M", + "timeShift": null, + "title": "S3 API Monthly Cost if on AWS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": "Cost in US$", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "currencyUSD", + "label": "Write Cost", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "S3 Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 252, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le, exported_instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Server Request Duration 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(SeaweedFS_volumeServer_request_total[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Server QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "fill": 1, + "id": 48, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "expr": "sum(SeaweedFS_volumeServer_max_volumes)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "fill": 1, + "id": 50, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Used Disk Space by Collection and Type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "fill": 1, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (exported_instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Used Disk Space by Host", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "fill": 1, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(push_time_seconds{} - (push_time_seconds{} offset 15s)) / 15", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Heartbeat by Host (Push Metrics Delta)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "Online", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Volume Server", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 251, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Store Request Duration 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(SeaweedFS_filerStore_request_total [1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Store QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Filer Store", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 242, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 52, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{exported_job=\"filer\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "bytes allocated", + "refId": "B" + }, + { + "expr": "rate(go_memstats_alloc_bytes_total{exported_job=\"filer\"}[30s])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "alloc rate", + "refId": "A" + }, + { + "expr": "go_memstats_stack_inuse_bytes{exported_job=\"filer\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "stack inuse", + "refId": "C" + }, + { + "expr": "go_memstats_heap_inuse_bytes{exported_job=\"filer\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "heap inuse", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Go Memory Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 54, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{exported_job=\"filer\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Go GC duration quantiles", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS-DEV}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 53, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{exported_job=\"filer\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filer Go Routines", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Filer Instances", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "SeaweedFS ks8 heartbeat", + "version": 1 +} \ No newline at end of file diff --git a/test/s3/basic/basic_test.go b/test/s3/basic/basic_test.go index 653fa1237..b2a6ae14b 100644 --- a/test/s3/basic/basic_test.go +++ b/test/s3/basic/basic_test.go @@ -2,14 +2,15 @@ package basic import ( "fmt" + "io" + "os" + "strings" + "testing" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "io/ioutil" - "os" - "strings" - "testing" ) var ( @@ -108,8 +109,8 @@ func TestListBucket(t *testing.T) { func TestListObjectV2(t *testing.T) { listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(Bucket), - Prefix: aws.String("foo"), + Bucket: aws.String(Bucket), + Prefix: aws.String("foo"), Delimiter: aws.String("/"), }) if err != nil { @@ -169,7 +170,7 @@ func TestObjectOp(t *testing.T) { exitErrorf("Unable to get copy object, %v", err) } - data, err := ioutil.ReadAll(getObj.Body) + data, err := io.ReadAll(getObj.Body) if err != nil { exitErrorf("Unable to read object data, %v", err) } diff --git a/test/s3/compatibility/.gitignore b/test/s3/compatibility/.gitignore new file mode 100644 index 000000000..dc3cc5207 --- /dev/null +++ b/test/s3/compatibility/.gitignore @@ -0,0 +1,2 @@ +/s3-tests +/tmp diff --git a/test/s3/compatibility/Dockerfile b/test/s3/compatibility/Dockerfile new file mode 100644 index 000000000..b2a1040cb --- /dev/null +++ b/test/s3/compatibility/Dockerfile @@ -0,0 +1,11 @@ +# the tests only support python 3.6, not newer +FROM ubuntu:latest + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata +RUN git clone https://github.com/ceph/s3-tests.git +WORKDIR s3-tests + +# we pin a certain commit +RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e + +RUN ./bootstrap diff --git a/test/s3/compatibility/README.md b/test/s3/compatibility/README.md new file mode 100644 index 000000000..de1b6e9ec --- /dev/null +++ b/test/s3/compatibility/README.md @@ -0,0 +1,13 @@ +# Running S3 Compatibility tests against SeaweedFS + +This is using [the tests from CephFS](https://github.com/ceph/s3-tests). + +## Prerequisites + +- have Docker installed +- this has been executed on Mac. On Linux, the hostname in `s3tests.conf` needs to be adjusted. + +## Running tests + +- `./prepare.sh` to build the docker image +- `./run.sh` to execute all tests diff --git a/test/s3/compatibility/prepare.sh b/test/s3/compatibility/prepare.sh new file mode 100755 index 000000000..7f9c20746 --- /dev/null +++ b/test/s3/compatibility/prepare.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +docker build --progress=plain -t s3tests . diff --git a/test/s3/compatibility/run.sh b/test/s3/compatibility/run.sh new file mode 100755 index 000000000..990599df5 --- /dev/null +++ b/test/s3/compatibility/run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ex + +killall -9 weed || echo "already stopped" +rm -Rf tmp +mkdir tmp +docker stop s3test-instance || echo "already stopped" + +ulimit -n 10000 +../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log & + +until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do + printf '.' + sleep 5 +done +sleep 3 + +rm -Rf logs-full.txt logs-summary.txt +# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v -a 'resource=object,!bucket-policy,!versioning,!encryption' +docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt + +docker stop s3test-instance || echo "already stopped" +killall -9 weed diff --git a/test/s3/compatibility/s3tests.conf b/test/s3/compatibility/s3tests.conf new file mode 100644 index 000000000..5adb61791 --- /dev/null +++ b/test/s3/compatibility/s3tests.conf @@ -0,0 +1,109 @@ +[DEFAULT] +## this section is just used for host, port and bucket_prefix + +# host set for rgw in vstart.sh +host = host.docker.internal + +# port set for rgw in vstart.sh +port = 8333 + +## say "False" to disable TLS +is_secure = False + +## say "False" to disable SSL Verify +ssl_verify = False + +[fixtures] +## all the buckets created will start with this prefix; +## {random} will be filled with random characters to pad +## the prefix to 30 characters long, and avoid collisions +bucket prefix = yournamehere-{random}- + +[s3 main] +# main display_name set in vstart.sh +display_name = M. Tester + +# main user_idname set in vstart.sh +user_id = testid + +# main email set in vstart.sh +email = tester@ceph.com + +# zonegroup api_name for bucket location +api_name = default + +## main AWS access key +access_key = 0555b35654ad1656d804 + +## main AWS secret key +secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== + +## replace with key id obtained when secret is created, or delete if KMS not tested +#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef + +[s3 alt] +# alt display_name set in vstart.sh +display_name = john.doe +## alt email set in vstart.sh +email = john.doe@example.com + +# alt user_id set in vstart.sh +user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 + +# alt AWS access key set in vstart.sh +access_key = NOPQRSTUVWXYZABCDEFG + +# alt AWS secret key set in vstart.sh +secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm + +[s3 tenant] +# tenant display_name set in vstart.sh +display_name = testx$tenanteduser + +# tenant user_id set in vstart.sh +user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +# tenant AWS secret key set in vstart.sh +access_key = HIJKLMNOPQRSTUVWXYZA + +# tenant AWS secret key set in vstart.sh +secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab + +# tenant email set in vstart.sh +email = tenanteduser@example.com + +#following section needs to be added for all sts-tests +[iam] +#used for iam operations in sts-tests +#email from vstart.sh +email = s3@example.com + +#user_id from vstart.sh +user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +#access_key from vstart.sh +access_key = ABCDEFGHIJKLMNOPQRST + +#secret_key vstart.sh +secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn + +#display_name from vstart.sh +display_name = youruseridhere + +#following section needs to be added when you want to run Assume Role With Webidentity test +[webidentity] +#used for assume role with web identity test in sts-tests +#all parameters will be obtained from ceph/qa/tasks/keycloak.py +token= + +aud= + +sub= + +azp= + +user_token=] + +thumbprint= + +KC_REALM= diff --git a/test/s3/multipart/aws_upload.go b/test/s3/multipart/aws_upload.go index 8c15cf6ed..0553bd403 100644 --- a/test/s3/multipart/aws_upload.go +++ b/test/s3/multipart/aws_upload.go @@ -19,10 +19,10 @@ import ( const ( maxPartSize = int64(5 * 1024 * 1024) maxRetries = 3 - awsAccessKeyID = "Your access key" - awsSecretAccessKey = "Your secret key" - awsBucketRegion = "S3 bucket region" - awsBucketName = "newBucket" + awsAccessKeyID = "any" + awsSecretAccessKey = "any" + awsBucketRegion = "us‑west‑1" + awsBucketName = "bucket1" ) var ( @@ -37,7 +37,7 @@ func main() { if err != nil { fmt.Printf("bad credentials: %s", err) } - cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333") + cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333").WithS3ForcePathStyle(true) svc := s3.New(session.New(), cfg) file, err := os.Open(*filename) diff --git a/unmaintained/diff_volume_servers/diff_volume_servers.go b/unmaintained/diff_volume_servers/diff_volume_servers.go index 27a537617..0188d18d4 100644 --- a/unmaintained/diff_volume_servers/diff_volume_servers.go +++ b/unmaintained/diff_volume_servers/diff_volume_servers.go @@ -6,13 +6,9 @@ import ( "errors" "flag" "fmt" - "io" - "math" - "os" - "strings" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage/idx" @@ -20,6 +16,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" "google.golang.org/grpc" + "io" + "math" + "os" ) var ( @@ -45,13 +44,13 @@ func main() { grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") vid := uint32(*volumeId) - servers := strings.Split(*serversStr, ",") + servers := pb.ServerAddresses(*serversStr).ToAddresses() if len(servers) < 2 { glog.Fatalf("You must specify more than 1 server\n") } - var referenceServer string + var referenceServer pb.ServerAddress var maxOffset int64 - allFiles := map[string]map[types.NeedleId]needleState{} + allFiles := map[pb.ServerAddress]map[types.NeedleId]needleState{} for _, addr := range servers { files, offset, err := getVolumeFiles(vid, addr) if err != nil { @@ -121,9 +120,9 @@ type needleState struct { size types.Size } -func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int64, error) { +func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleState, int64, error) { var idxFile *bytes.Reader - err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() copyFileClient, err := vs.CopyFile(ctx, &volume_server_pb.CopyFileRequest{ @@ -179,9 +178,9 @@ func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int6 return files, maxOffset, nil } -func getNeedleFileId(v uint32, nid types.NeedleId, addr string) (string, error) { +func getNeedleFileId(v uint32, nid types.NeedleId, addr pb.ServerAddress) (string, error) { var id string - err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { resp, err := vs.VolumeNeedleStatus(context.Background(), &volume_server_pb.VolumeNeedleStatusRequest{ VolumeId: v, NeedleId: uint64(nid), diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index 70bce3bf9..457c5c592 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -24,7 +24,7 @@ var ( /* This is to resolve an one-time issue that caused inconsistency with .dat and .idx files. -In this case, the .dat file contains all data, but some of deletion caused incorrect offset. +In this case, the .dat file contains all data, but some deletion caused incorrect offset. The .idx has all correct offsets. 1. fix the .dat file, a new .dat_fixed file will be generated. diff --git a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go index c521ce33e..d4b9d63b1 100644 --- a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go +++ b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go @@ -12,10 +12,11 @@ import ( ) var ( - dir = flag.String("dir", "/tmp", "directory to create files") - n = flag.Int("n", 100, "the number of metadata") - tailFiler = flag.String("filer", "localhost:8888", "the filer address") - isWrite = flag.Bool("write", false, "only write") + dir = flag.String("dir", "/tmp", "directory to create files") + n = flag.Int("n", 100, "the number of metadata") + tailFiler = flag.String("filer", "localhost:8888", "the filer address") + isWrite = flag.Bool("write", false, "only write") + writeInterval = flag.Duration("writeInterval", 0, "write interval, e.g., 1s") ) func main() { @@ -50,10 +51,11 @@ func main() { } func startGenerateMetadata() { - pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { + pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { for i := 0; i < *n; i++ { name := fmt.Sprintf("file%d", i) + glog.V(0).Infof("write %s/%s", *dir, name) if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ Directory: *dir, Entry: &filer_pb.Entry{ @@ -63,6 +65,9 @@ func startGenerateMetadata() { fmt.Printf("create entry %s: %v\n", name, err) return err } + if *writeInterval > 0 { + time.Sleep(*writeInterval) + } } return nil @@ -72,8 +77,7 @@ func startGenerateMetadata() { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { - tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", - *dir, 0, 0, eachEntryFunc, false) + tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError) if tailErr != nil { fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index d85e45af0..937e764bb 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "log" "math/rand" "time" @@ -32,7 +33,7 @@ func main() { go func() { for { println("vacuum threshold", *garbageThreshold) - _, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", *master, *garbageThreshold)) + _, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold)) if err != nil { log.Fatalf("vacuum: %v", err) } @@ -52,7 +53,7 @@ func main() { } func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) { - assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{ + assignResult, err := operation.Assign(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{ Count: 1, Replication: *replication, }) @@ -65,7 +66,16 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) - _, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth) + uploadOption := &operation.UploadOption{ + UploadUrl: targetUrl, + Filename: fmt.Sprintf("test%d", i), + Cipher: false, + IsInputCompressed: true, + MimeType: "bench/test", + PairMap: nil, + Jwt: assignResult.Auth, + } + _, err = operation.UploadData(data, uploadOption) if err != nil { log.Fatalf("upload: %v", err) } diff --git a/unmaintained/stream_read_volume/stream_read_volume.go b/unmaintained/stream_read_volume/stream_read_volume.go new file mode 100644 index 000000000..bbe5abedb --- /dev/null +++ b/unmaintained/stream_read_volume/stream_read_volume.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" + "io" +) + +var ( + volumeServer = flag.String("volumeServer", "localhost:8080", "a volume server") + volumeId = flag.Int("volumeId", -1, "a volume id to stream read") + grpcDialOption grpc.DialOption +) + +func main() { + flag.Parse() + + util.LoadConfiguration("security", false) + grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") + + vid := uint32(*volumeId) + + eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error { + fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob)) + return nil + } + + err := operation.WithVolumeServerClient(true, pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{ + VolumeIds: []uint32{vid}, + }) + if err != nil { + return err + } + for { + resp, err := copyFileClient.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if err = eachNeedleFunc(resp); err != nil { + return err + } + } + return nil + }) + if err != nil { + fmt.Printf("read %s: %v\n", *volumeServer, err) + } + +} diff --git a/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go b/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go index 2ee8028f2..a41bf1da1 100644 --- a/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go +++ b/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "math/rand" "mime/multipart" @@ -45,7 +44,7 @@ func main() { defer wg.Done() client := &http.Client{Transport: &http.Transport{ - MaxIdleConns: 1024, + MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, }} r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x))) @@ -128,7 +127,7 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s if err != nil { return 0, fmt.Errorf("read http POST %s response: %v", uri, err) } - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } diff --git a/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go b/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go index 8b986b546..83df54dc3 100644 --- a/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go +++ b/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "math/rand" "mime/multipart" @@ -36,7 +35,7 @@ func main() { var fileNames []string - files, err := ioutil.ReadDir(*dir) + files, err := os.ReadDir(*dir) if err != nil { log.Fatalf("fail to read dir %v: %v", *dir, err) } @@ -142,7 +141,7 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size if err != nil { return 0, fmt.Errorf("read http POST %s response: %v", uri, err) } - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() } diff --git a/unmaintained/volume_tailer/volume_tailer.go b/unmaintained/volume_tailer/volume_tailer.go index 32da2e6ab..3c6f4a987 100644 --- a/unmaintained/volume_tailer/volume_tailer.go +++ b/unmaintained/volume_tailer/volume_tailer.go @@ -2,6 +2,7 @@ package main import ( "flag" + "github.com/chrislusf/seaweedfs/weed/pb" "log" "time" @@ -37,7 +38,7 @@ func main() { sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano() } - err := operation.TailVolume(func()string{return *master}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) { + err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) { if n.Size == 0 { println("-", n.String()) return nil diff --git a/weed/Makefile b/weed/Makefile index 46034977e..4e871a71e 100644 --- a/weed/Makefile +++ b/weed/Makefile @@ -2,10 +2,13 @@ BINARY = weed SOURCE_DIR = . -all: debug_mount +all: install .PHONY : clean debug_mount +install: + go install + clean: go clean $(SOURCE_DIR) rm -f $(BINARY) @@ -16,15 +19,15 @@ debug_shell: debug_mount: go build -gcflags="all=-N -l" - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000 debug_server: go build -gcflags="all=-N -l" - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 debug_volume: go build -gcflags="all=-N -l" - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=/Volumes/mobile_disk/100 -port 8564 -max=30 -preStopSeconds=2 + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2 debug_webdav: go build -gcflags="all=-N -l" @@ -38,11 +41,14 @@ debug_filer_copy: go build -gcflags="all=-N -l" dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h -debug_filer_remote_sync: +debug_filer_remote_sync_dir: go build -gcflags="all=-N -l" dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h +debug_filer_remote_sync_buckets: + go build -gcflags="all=-N -l" + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -createBucketAt=cloud1 -timeAgo=1h + debug_master_follower: go build -gcflags="all=-N -l" dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower - diff --git a/weed/cluster/cluster.go b/weed/cluster/cluster.go new file mode 100644 index 000000000..6c24df44c --- /dev/null +++ b/weed/cluster/cluster.go @@ -0,0 +1,315 @@ +package cluster + +import ( + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "math" + "sync" + "time" +) + +const ( + MasterType = "master" + VolumeServerType = "volumeServer" + FilerType = "filer" + BrokerType = "broker" +) + +type FilerGroup string +type Filers struct { + filers map[pb.ServerAddress]*ClusterNode + leaders *Leaders +} +type Leaders struct { + leaders [3]pb.ServerAddress +} + +type ClusterNode struct { + Address pb.ServerAddress + Version string + counter int + CreatedTs time.Time +} + +type Cluster struct { + filerGroup2filers map[FilerGroup]*Filers + filersLock sync.RWMutex + brokers map[pb.ServerAddress]*ClusterNode + brokersLock sync.RWMutex +} + +func NewCluster() *Cluster { + return &Cluster{ + filerGroup2filers: make(map[FilerGroup]*Filers), + brokers: make(map[pb.ServerAddress]*ClusterNode), + } +} + +func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers { + cluster.filersLock.Lock() + defer cluster.filersLock.Unlock() + filers, found := cluster.filerGroup2filers[filerGroup] + if !found && createIfNotFound { + filers = &Filers{ + filers: make(map[pb.ServerAddress]*ClusterNode), + leaders: &Leaders{}, + } + cluster.filerGroup2filers[filerGroup] = filers + } + return filers +} + +func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse { + filerGroup := FilerGroup(ns) + switch nodeType { + case FilerType: + filers := cluster.getFilers(filerGroup, true) + if existingNode, found := filers.filers[address]; found { + existingNode.counter++ + return nil + } + filers.filers[address] = &ClusterNode{ + Address: address, + Version: version, + counter: 1, + CreatedTs: time.Now(), + } + return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address) + case BrokerType: + cluster.brokersLock.Lock() + defer cluster.brokersLock.Unlock() + if existingNode, found := cluster.brokers[address]; found { + existingNode.counter++ + return nil + } + cluster.brokers[address] = &ClusterNode{ + Address: address, + Version: version, + counter: 1, + CreatedTs: time.Now(), + } + return []*master_pb.KeepConnectedResponse{ + { + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + NodeType: nodeType, + Address: string(address), + IsAdd: true, + }, + }, + } + case MasterType: + return []*master_pb.KeepConnectedResponse{ + { + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + NodeType: nodeType, + Address: string(address), + IsAdd: true, + }, + }, + } + } + return nil +} + +func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse { + filerGroup := FilerGroup(ns) + switch nodeType { + case FilerType: + filers := cluster.getFilers(filerGroup, false) + if filers == nil { + return nil + } + if existingNode, found := filers.filers[address]; !found { + return nil + } else { + existingNode.counter-- + if existingNode.counter <= 0 { + delete(filers.filers, address) + return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address) + } + } + case BrokerType: + cluster.brokersLock.Lock() + defer cluster.brokersLock.Unlock() + if existingNode, found := cluster.brokers[address]; !found { + return nil + } else { + existingNode.counter-- + if existingNode.counter <= 0 { + delete(cluster.brokers, address) + return []*master_pb.KeepConnectedResponse{ + { + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + NodeType: nodeType, + Address: string(address), + IsAdd: false, + }, + }, + } + } + } + case MasterType: + return []*master_pb.KeepConnectedResponse{ + { + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + NodeType: nodeType, + Address: string(address), + IsAdd: false, + }, + }, + } + } + return nil +} + +func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) { + switch nodeType { + case FilerType: + filers := cluster.getFilers(filerGroup, false) + if filers == nil { + return + } + cluster.filersLock.RLock() + defer cluster.filersLock.RUnlock() + for _, node := range filers.filers { + nodes = append(nodes, node) + } + case BrokerType: + cluster.brokersLock.RLock() + defer cluster.brokersLock.RUnlock() + for _, node := range cluster.brokers { + nodes = append(nodes, node) + } + case MasterType: + } + return +} + +func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool { + filers := cluster.getFilers(filerGroup, false) + if filers == nil { + return false + } + return filers.leaders.isOneLeader(address) +} + +func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) { + if isAdd { + if filers.leaders.addLeaderIfVacant(address) { + // has added the address as one leader + result = append(result, &master_pb.KeepConnectedResponse{ + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + FilerGroup: string(filerGroup), + NodeType: nodeType, + Address: string(address), + IsLeader: true, + IsAdd: true, + }, + }) + } else { + result = append(result, &master_pb.KeepConnectedResponse{ + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + FilerGroup: string(filerGroup), + NodeType: nodeType, + Address: string(address), + IsLeader: false, + IsAdd: true, + }, + }) + } + } else { + if filers.leaders.removeLeaderIfExists(address) { + + result = append(result, &master_pb.KeepConnectedResponse{ + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + FilerGroup: string(filerGroup), + NodeType: nodeType, + Address: string(address), + IsLeader: true, + IsAdd: false, + }, + }) + + // pick the freshest one, since it is less likely to go away + var shortestDuration int64 = math.MaxInt64 + now := time.Now() + var candidateAddress pb.ServerAddress + for _, node := range filers.filers { + if filers.leaders.isOneLeader(node.Address) { + continue + } + duration := now.Sub(node.CreatedTs).Nanoseconds() + if duration < shortestDuration { + shortestDuration = duration + candidateAddress = node.Address + } + } + if candidateAddress != "" { + filers.leaders.addLeaderIfVacant(candidateAddress) + // added a new leader + result = append(result, &master_pb.KeepConnectedResponse{ + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + NodeType: nodeType, + Address: string(candidateAddress), + IsLeader: true, + IsAdd: true, + }, + }) + } + } else { + result = append(result, &master_pb.KeepConnectedResponse{ + ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ + FilerGroup: string(filerGroup), + NodeType: nodeType, + Address: string(address), + IsLeader: false, + IsAdd: false, + }, + }) + } + } + return +} + +func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) { + if leaders.isOneLeader(address) { + return + } + for i := 0; i < len(leaders.leaders); i++ { + if leaders.leaders[i] == "" { + leaders.leaders[i] = address + hasChanged = true + return + } + } + return +} +func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) { + if !leaders.isOneLeader(address) { + return + } + for i := 0; i < len(leaders.leaders); i++ { + if leaders.leaders[i] == address { + leaders.leaders[i] = "" + hasChanged = true + return + } + } + return +} +func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool { + for i := 0; i < len(leaders.leaders); i++ { + if leaders.leaders[i] == address { + return true + } + } + return false +} +func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) { + for i := 0; i < len(leaders.leaders); i++ { + if leaders.leaders[i] != "" { + addresses = append(addresses, leaders.leaders[i]) + } + } + return +} diff --git a/weed/cluster/cluster_test.go b/weed/cluster/cluster_test.go new file mode 100644 index 000000000..ccaccf6f7 --- /dev/null +++ b/weed/cluster/cluster_test.go @@ -0,0 +1,47 @@ +package cluster + +import ( + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestClusterAddRemoveNodes(t *testing.T) { + c := NewCluster() + + c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45") + c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45") + assert.Equal(t, []pb.ServerAddress{ + pb.ServerAddress("111:1"), + pb.ServerAddress("111:2"), + }, c.getFilers("", false).leaders.GetLeaders()) + + c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45") + c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45") + assert.Equal(t, []pb.ServerAddress{ + pb.ServerAddress("111:1"), + pb.ServerAddress("111:2"), + pb.ServerAddress("111:3"), + }, c.getFilers("", false).leaders.GetLeaders()) + + c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45") + c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45") + c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4")) + assert.Equal(t, []pb.ServerAddress{ + pb.ServerAddress("111:1"), + pb.ServerAddress("111:2"), + pb.ServerAddress("111:3"), + }, c.getFilers("", false).leaders.GetLeaders()) + + // remove oldest + c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1")) + assert.Equal(t, []pb.ServerAddress{ + pb.ServerAddress("111:6"), + pb.ServerAddress("111:2"), + pb.ServerAddress("111:3"), + }, c.getFilers("", false).leaders.GetLeaders()) + + // remove oldest + c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1")) + +} diff --git a/weed/command/autocomplete.go b/weed/command/autocomplete.go index 9a545a183..955ce4006 100644 --- a/weed/command/autocomplete.go +++ b/weed/command/autocomplete.go @@ -41,7 +41,7 @@ func AutocompleteMain(commands []*Command) bool { func installAutoCompletion() bool { if runtime.GOOS == "windows" { - fmt.Println("windows is not supported") + fmt.Println("Windows is not supported") return false } @@ -56,7 +56,7 @@ func installAutoCompletion() bool { func uninstallAutoCompletion() bool { if runtime.GOOS == "windows" { - fmt.Println("windows is not supported") + fmt.Println("Windows is not supported") return false } @@ -65,7 +65,7 @@ func uninstallAutoCompletion() bool { fmt.Printf("uninstall failed! %s\n", err) return false } - fmt.Printf("autocompletion is disable. Please restart your shell.\n") + fmt.Printf("autocompletion is disabled. Please restart your shell.\n") return true } diff --git a/weed/command/backup.go b/weed/command/backup.go index 4c5a2d820..c43b0d351 100644 --- a/weed/command/backup.go +++ b/weed/command/backup.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -72,12 +73,12 @@ func runBackup(cmd *Command, args []string) bool { vid := needle.VolumeId(*s.volumeId) // find volume location, replication, ttl info - lookup, err := operation.LookupVolumeId(func() string { return *s.master }, grpcDialOption, vid.String()) + lookup, err := operation.LookupVolumeId(func() pb.ServerAddress { return pb.ServerAddress(*s.master) }, grpcDialOption, vid.String()) if err != nil { fmt.Printf("Error looking up volume %d: %v\n", vid, err) return true } - volumeServer := lookup.Locations[0].Url + volumeServer := lookup.Locations[0].ServerAddress() stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid)) if err != nil { @@ -119,7 +120,7 @@ func runBackup(cmd *Command, args []string) bool { } if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) { - if err = v.Compact2(30*1024*1024*1024, 0); err != nil { + if err = v.Compact2(0, 0, nil); err != nil { fmt.Printf("Compact Volume before synchronizing %v\n", err) return true } diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index f0c8f6139..9f18cc5b9 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -3,6 +3,7 @@ package command import ( "bufio" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "math" "math/rand" @@ -10,7 +11,6 @@ import ( "runtime" "runtime/pprof" "sort" - "strings" "sync" "time" @@ -74,14 +74,14 @@ func init() { var cmdBenchmark = &Command{ UsageLine: "benchmark -master=localhost:9333 -c=10 -n=100000", - Short: "benchmark on writing millions of files and read out", + Short: "benchmark by writing millions of files and reading them out", Long: `benchmark on an empty SeaweedFS file system. Two tests during benchmark: 1) write lots of small files to the system 2) read the files out - The file content is mostly zero, but no compression is done. + The file content is mostly zeros, but no compression is done. You can choose to only benchmark read or write. During write, the list of uploaded file ids is stored in "-list" specified file. @@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool { defer pprof.StopCPUProfile() } - b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", 0, "", strings.Split(*b.masters, ",")) + b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap()) go b.masterClient.KeepConnectedToMaster() b.masterClient.WaitUntilConnected() @@ -468,7 +468,7 @@ func (s *stats) printStats() { timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000 fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency) fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken) - fmt.Printf("Complete requests: %d\n", completed) + fmt.Printf("Completed requests: %d\n", completed) fmt.Printf("Failed requests: %d\n", failed) fmt.Printf("Total transferred: %d bytes\n", transferred) fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken) diff --git a/weed/command/command.go b/weed/command/command.go index 8d6525652..7635405dc 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -2,9 +2,10 @@ package command import ( "fmt" - flag "github.com/chrislusf/seaweedfs/weed/util/fla9" "os" "strings" + + flag "github.com/chrislusf/seaweedfs/weed/util/fla9" ) var Commands = []*Command{ @@ -21,6 +22,7 @@ var Commands = []*Command{ cmdFilerCopy, cmdFilerMetaBackup, cmdFilerMetaTail, + cmdFilerRemoteGateway, cmdFilerRemoteSynchronize, cmdFilerReplicate, cmdFilerSynchronize, @@ -35,6 +37,7 @@ var Commands = []*Command{ cmdScaffold, cmdServer, cmdShell, + cmdUpdate, cmdUpload, cmdVersion, cmdVolume, diff --git a/weed/command/compact.go b/weed/command/compact.go index 92e25f474..6df28440a 100644 --- a/weed/command/compact.go +++ b/weed/command/compact.go @@ -50,7 +50,7 @@ func runCompact(cmd *Command, args []string) bool { glog.Fatalf("Compact Volume [ERROR] %s\n", err) } } else { - if err = v.Compact2(preallocate, 0); err != nil { + if err = v.Compact2(preallocate, 0, nil); err != nil { glog.Fatalf("Compact Volume [ERROR] %s\n", err) } } diff --git a/weed/command/download.go b/weed/command/download.go index a64d3f237..a3c05b53d 100644 --- a/weed/command/download.go +++ b/weed/command/download.go @@ -2,16 +2,17 @@ package command import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/security" - "google.golang.org/grpc" "io" - "io/ioutil" "net/http" "os" "path" "strings" + "google.golang.org/grpc" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -49,7 +50,7 @@ func runDownload(cmd *Command, args []string) bool { grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for _, fid := range args { - if e := downloadToFile(func() string { return *d.server }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil { + if e := downloadToFile(func() pb.ServerAddress { return pb.ServerAddress(*d.server) }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil { fmt.Println("Download Error: ", fid, e) } } @@ -81,7 +82,7 @@ func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpti } defer f.Close() if isFileList { - content, err := ioutil.ReadAll(rc.Body) + content, err := io.ReadAll(rc.Body) if err != nil { return err } @@ -118,7 +119,7 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption return "", nil, e } defer util.CloseResponse(rc) - content, e = ioutil.ReadAll(rc.Body) + content, e = io.ReadAll(rc.Body) return } diff --git a/weed/command/filer.go b/weed/command/filer.go index ddee0852c..c9f9a1956 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -2,10 +2,10 @@ package command import ( "fmt" + "net" "net/http" "os" - "strconv" - "strings" + "runtime" "time" "google.golang.org/grpc/reflection" @@ -14,7 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/server" + weed_server "github.com/chrislusf/seaweedfs/weed/server" stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -30,11 +30,14 @@ var ( ) type FilerOptions struct { - masters *string + masters map[string]pb.ServerAddress + mastersString *string ip *string bindIp *string port *int + portGrpc *int publicPort *int + filerGroup *string collection *string defaultReplicaPlacement *string disableDirListing *bool @@ -45,22 +48,25 @@ type FilerOptions struct { enableNotification *bool disableHttp *bool cipher *bool - peers *string metricsHttpPort *int saveToFilerLimit *int defaultLevelDbDirectory *string concurrentUploadLimitMB *int debug *bool debugPort *int + localSocket *string + showUIDirectoryDelete *bool } func init() { cmdFiler.Run = runFiler // break init cycle - f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") + f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") + f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup") f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection") f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address") - f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to") + f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") + f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port") f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public") f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") @@ -70,22 +76,26 @@ func init() { f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack") f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed") f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers") - f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list") f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store") f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory") f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size") f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging") + f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-.sock") + f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port") + filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port") filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file") filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file") filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file") - filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") + filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") + filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders") + filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") // start webdav on filer filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") @@ -96,10 +106,11 @@ func init() { filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") - filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") // start iam on filer filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service") + filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address") filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port") } @@ -134,10 +145,12 @@ func runFiler(cmd *Command, args []string) bool { go stats_collect.StartMetricsServer(*f.metricsHttpPort) - filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port) + filerAddress := util.JoinHostPort(*f.ip, *f.port) startDelay := time.Duration(2) if *filerStartS3 { filerS3Options.filer = &filerAddress + filerS3Options.bindIp = f.bindIp + filerS3Options.localFilerSocket = f.localSocket go func() { time.Sleep(startDelay * time.Second) filerS3Options.startS3Server() @@ -156,13 +169,15 @@ func runFiler(cmd *Command, args []string) bool { if *filerStartIam { filerIamOptions.filer = &filerAddress - filerIamOptions.masters = f.masters + filerIamOptions.masters = f.mastersString go func() { time.Sleep(startDelay * time.Second) filerIamOptions.startIamServer() }() } + f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap() + f.startFiler() return true @@ -176,16 +191,20 @@ func (fo *FilerOptions) startFiler() { if *fo.publicPort != 0 { publicVolumeMux = http.NewServeMux() } + if *fo.portGrpc == 0 { + *fo.portGrpc = 10000 + *fo.port + } + if *fo.bindIp == "" { + *fo.bindIp = *fo.ip + } defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2") - var peers []string - if *fo.peers != "" { - peers = strings.Split(*fo.peers, ",") - } + filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc) fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{ - Masters: strings.Split(*fo.masters, ","), + Masters: fo.masters, + FilerGroup: *fo.filerGroup, Collection: *fo.collection, DefaultReplication: *fo.defaultReplicaPlacement, DisableDirListing: *fo.disableDirListing, @@ -195,21 +214,20 @@ func (fo *FilerOptions) startFiler() { Rack: *fo.rack, DefaultLevelDbDir: defaultLevelDbDirectory, DisableHttp: *fo.disableHttp, - Host: *fo.ip, - Port: uint32(*fo.port), + Host: filerAddress, Cipher: *fo.cipher, SaveToFilerLimit: int64(*fo.saveToFilerLimit), - Filers: peers, ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024, + ShowUIDirectoryDelete: *fo.showUIDirectoryDelete, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) } if *fo.publicPort != 0 { - publicListeningAddress := *fo.bindIp + ":" + strconv.Itoa(*fo.publicPort) + publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort) glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress) - publicListener, e := util.NewListener(publicListeningAddress, 0) + publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0) if e != nil { glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e) } @@ -218,11 +236,18 @@ func (fo *FilerOptions) startFiler() { glog.Fatalf("Volume server fail to serve public: %v", e) } }() + if localPublicListner != nil { + go func() { + if e := http.Serve(localPublicListner, publicVolumeMux); e != nil { + glog.Errorf("Volume server fail to serve public: %v", e) + } + }() + } } glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port) - filerListener, e := util.NewListener( - *fo.bindIp+":"+strconv.Itoa(*fo.port), + filerListener, filerLocalListener, e := util.NewIpAndLocalListeners( + *fo.bindIp, *fo.port, time.Duration(10)*time.Second, ) if e != nil { @@ -230,17 +255,43 @@ func (fo *FilerOptions) startFiler() { } // starting grpc server - grpcPort := *fo.port + 10000 - grpcL, err := util.NewListener(*fo.bindIp+":"+strconv.Itoa(grpcPort), 0) + grpcPort := *fo.portGrpc + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0) if err != nil { glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err) } grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer")) filer_pb.RegisterSeaweedFilerServer(grpcS, fs) reflection.Register(grpcS) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } go grpcS.Serve(grpcL) httpS := &http.Server{Handler: defaultMux} + if runtime.GOOS != "windows" { + if *fo.localSocket == "" { + *fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port) + } + if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) { + glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error()) + } + go func() { + // start on local unix socket + filerSocketListener, err := net.Listen("unix", *fo.localSocket) + if err != nil { + glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err) + } + httpS.Serve(filerSocketListener) + }() + } + if filerLocalListener != nil { + go func() { + if err := httpS.Serve(filerLocalListener); err != nil { + glog.Errorf("Filer Fail to serve: %v", e) + } + }() + } if err := httpS.Serve(filerListener); err != nil { glog.Fatalf("Filer Fail to serve: %v", e) } diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 0c450181b..d191c693b 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -54,8 +54,10 @@ func runFilerBackup(cmd *Command, args []string) bool { grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + clientId := util.RandomInt32() + for { - err := doFilerBackup(grpcDialOption, &filerBackupOptions) + err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId) if err != nil { glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) time.Sleep(1747 * time.Millisecond) @@ -69,7 +71,7 @@ const ( BackupKeyPrefix = "backup." ) -func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error { +func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32) error { // find data sink config := util.GetViper() @@ -78,7 +80,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti return fmt.Errorf("no data sink configured in replication.toml") } - sourceFiler := *backupOption.filer + sourceFiler := pb.ServerAddress(*backupOption.filer) sourcePath := *backupOption.path timeAgo := *backupOption.timeAgo targetPath := dataSink.GetSinkToDirectory() @@ -102,7 +104,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti // create filer sink filerSource := &source.FilerSource{} - filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler) + filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler) dataSink.SetSourceFiler(filerSource) processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug) @@ -112,7 +114,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs) }) - return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), - sourcePath, startFrom.UnixNano(), 0, processEventFnWithOffset, false) + return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError) } diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go index 09f5e97fe..ada843dea 100644 --- a/weed/command/filer_cat.go +++ b/weed/command/filer_cat.go @@ -8,7 +8,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/wdclient" "google.golang.org/grpc" - "math" "net/url" "os" "strings" @@ -23,7 +22,7 @@ var ( type FilerCatOptions struct { grpcDialOption grpc.DialOption - filerAddress string + filerAddress pb.ServerAddress filerClient filer_pb.SeaweedFilerClient output *string } @@ -78,7 +77,7 @@ func runFilerCat(cmd *Command, args []string) bool { return false } - filerCat.filerAddress = filerUrl.Host + filerCat.filerAddress = pb.ServerAddress(filerUrl.Host) filerCat.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") dir, name := util.FullPath(urlPath).DirAndName() @@ -97,7 +96,7 @@ func runFilerCat(cmd *Command, args []string) bool { writer = f } - pb.WithFilerClient(filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Name: name, @@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool { filerCat.filerClient = client - return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry))) }) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 818ae5f23..f20ae99bf 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -3,11 +3,8 @@ package command import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" "io" - "io/ioutil" "net/http" - "net/url" "os" "path/filepath" "strconv" @@ -17,14 +14,14 @@ import ( "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/util/grace" - + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/chrislusf/seaweedfs/weed/wdclient" ) @@ -74,7 +71,7 @@ var cmdFilerCopy = &Command{ It can copy one or a list of files or folders. If copying a whole folder recursively: - All files under the folder and subfolders will be copyed. + All files under the folder and sub folders will be copied. Optional parameter "-include" allows you to specify the file name patterns. If "maxMB" is set to a positive number, files larger than it would be split into chunks. @@ -92,35 +89,21 @@ func runCopy(cmd *Command, args []string) bool { filerDestination := args[len(args)-1] fileOrDirs := args[0 : len(args)-1] - filerUrl, err := url.Parse(filerDestination) + filerAddress, urlPath, err := pb.ParseUrl(filerDestination) if err != nil { fmt.Printf("The last argument should be a URL on filer: %v\n", err) return false } - urlPath := filerUrl.Path if !strings.HasSuffix(urlPath, "/") { fmt.Printf("The last argument should be a folder and end with \"/\"\n") return false } - if filerUrl.Port() == "" { - fmt.Printf("The filer port should be specified.\n") - return false - } - - filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64) - if parseErr != nil { - fmt.Printf("The filer port parse error: %v\n", parseErr) - return false - } - - filerGrpcPort := filerPort + 10000 - filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort) copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") - masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress) + masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerAddress) if err != nil { - fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err) + fmt.Printf("read from filer %s: %v\n", filerAddress, err) return false } if strings.HasPrefix(urlPath, dirBuckets+"/") { @@ -174,9 +157,8 @@ func runCopy(cmd *Command, args []string) bool { go func() { defer waitGroup.Done() worker := FileCopyWorker{ - options: ©, - filerHost: filerUrl.Host, - filerGrpcAddress: filerGrpcAddress, + options: ©, + filerAddress: filerAddress, } if err := worker.copyFiles(fileCopyTaskChan); err != nil { fmt.Fprintf(os.Stderr, "copy file error: %v\n", err) @@ -189,8 +171,8 @@ func runCopy(cmd *Command, args []string) bool { return true } -func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) { - err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) { + err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) @@ -228,9 +210,9 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi } if mode.IsDir() { - files, _ := ioutil.ReadDir(fileOrDir) + files, _ := os.ReadDir(fileOrDir) for _, subFileOrDir := range files { - cleanedDestDirectory := filepath.Clean(destPath + fi.Name()) + cleanedDestDirectory := destPath + fi.Name() if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil { return err } @@ -241,9 +223,8 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi } type FileCopyWorker struct { - options *CopyOptions - filerHost string - filerGrpcAddress string + options *CopyOptions + filerAddress pb.ServerAddress } func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error { @@ -321,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi return } - err = pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Directory: task.destinationUrlPath, @@ -356,14 +337,14 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 { mimeType = detectMimeType(f) - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } - // assign a volume - err = util.Retry("assignVolume", func() error { - return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = util.Retry("upload", func() error { + // assign a volume + assignErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AssignVolumeRequest{ Count: 1, @@ -381,50 +362,62 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err if assignResult.Error != "" { return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error) } - if assignResult.Url == "" { + if assignResult.Location.Url == "" { return fmt.Errorf("assign volume failure %v: %v", request, assignResult) } return nil }) - }) - if err != nil { - return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err) - } + if assignErr != nil { + return assignErr + } + + // upload data + targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId + uploadOption := &operation.UploadOption{ + UploadUrl: targetUrl, + Filename: fileName, + Cipher: worker.options.cipher, + IsInputCompressed: false, + MimeType: mimeType, + PairMap: nil, + Jwt: security.EncodedJwt(assignResult.Auth), + } + uploadResult, err := operation.UploadData(data, uploadOption) + if err != nil { + return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) + } + if uploadResult.Error != "" { + return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) + } + if *worker.options.verbose { + fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) + } - targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId + fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName) + chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0)) - uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth)) + return nil + }) if err != nil { - return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) - } - if uploadResult.Error != "" { - return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) - } - if *worker.options.verbose { - fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) + return fmt.Errorf("upload %v: %v\n", fileName, err) } - chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0)) - - fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName) } - if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.CreateEntryRequest{ Directory: task.destinationUrlPath, Entry: &filer_pb.Entry{ Name: fileName, Attributes: &filer_pb.FuseAttributes{ - Crtime: time.Now().Unix(), - Mtime: time.Now().Unix(), - Gid: task.gid, - Uid: task.uid, - FileSize: uint64(task.fileSize), - FileMode: uint32(task.fileMode), - Mime: mimeType, - Replication: *worker.options.replication, - Collection: *worker.options.collection, - TtlSec: worker.options.ttlSec, + Crtime: time.Now().Unix(), + Mtime: time.Now().Unix(), + Gid: task.gid, + Uid: task.uid, + FileSize: uint64(task.fileSize), + FileMode: uint32(task.fileMode), + Mime: mimeType, + TtlSec: worker.options.ttlSec, }, Chunks: chunks, }, @@ -435,7 +428,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err } return nil }); err != nil { - return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err) + return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err) } return nil @@ -466,7 +459,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, var assignResult *filer_pb.AssignVolumeResponse var assignError error err := util.Retry("assignVolume", func() error { - return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AssignVolumeRequest{ Count: 1, Replication: *worker.options.replication, @@ -487,10 +480,11 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, }) }) if err != nil { - fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err) + uploadError = fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err) + return } - targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId + targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId if collection == "" { collection = assignResult.Collection } @@ -498,7 +492,16 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, replication = assignResult.Replication } - uploadResult, err, _ := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), worker.options.cipher, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth)) + uploadOption := &operation.UploadOption{ + UploadUrl: targetUrl, + Filename: fileName + "-" + strconv.FormatInt(i+1, 10), + Cipher: worker.options.cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: security.EncodedJwt(assignResult.Auth), + } + uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption) if err != nil { uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return @@ -525,8 +528,8 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, for _, chunk := range chunks { fileIds = append(fileIds, chunk.FileId) } - operation.DeleteFiles(func() string { - return copy.masters[0] + operation.DeleteFiles(func() pb.ServerAddress { + return pb.ServerAddress(copy.masters[0]) }, false, worker.options.grpcDialOption, fileIds) return uploadError } @@ -536,22 +539,20 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, return fmt.Errorf("create manifest: %v", manifestErr) } - if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.CreateEntryRequest{ Directory: task.destinationUrlPath, Entry: &filer_pb.Entry{ Name: fileName, Attributes: &filer_pb.FuseAttributes{ - Crtime: time.Now().Unix(), - Mtime: time.Now().Unix(), - Gid: task.gid, - Uid: task.uid, - FileSize: uint64(task.fileSize), - FileMode: uint32(task.fileMode), - Mime: mimeType, - Replication: replication, - Collection: collection, - TtlSec: worker.options.ttlSec, + Crtime: time.Now().Unix(), + Mtime: time.Now().Unix(), + Gid: task.gid, + Uid: task.uid, + FileSize: uint64(task.fileSize), + FileMode: uint32(task.fileMode), + Mime: mimeType, + TtlSec: worker.options.ttlSec, }, Chunks: manifestedChunks, }, @@ -562,10 +563,10 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, } return nil }); err != nil { - return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err) + return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err) } - fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName) + fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName) return nil } @@ -594,7 +595,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off var fileId, host string var auth security.EncodedJwt - if flushErr := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if flushErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { ctx := context.Background() @@ -616,7 +617,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off return fmt.Errorf("assign volume failure %v: %v", request, resp.Error) } - fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth) + fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth) collection, replication = resp.Collection, resp.Replication return nil @@ -630,8 +631,16 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr) } - fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - uploadResult, flushErr, _ := operation.Upload(fileUrl, name, worker.options.cipher, reader, false, "", nil, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId), + Filename: name, + Cipher: worker.options.cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + } + uploadResult, flushErr, _ := operation.Upload(reader, uploadOption) if flushErr != nil { return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr) } diff --git a/weed/command/filer_meta_backup.go b/weed/command/filer_meta_backup.go index 6fe323fba..cf679885d 100644 --- a/weed/command/filer_meta_backup.go +++ b/weed/command/filer_meta_backup.go @@ -27,7 +27,8 @@ type FilerMetaBackupOptions struct { restart *bool backupFilerConfig *string - store filer.FilerStore + store filer.FilerStore + clientId int32 } func init() { @@ -36,6 +37,7 @@ func init() { metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer") metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup") metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store") + metaBackup.clientId = util.RandomInt32() } var cmdFilerMetaBackup = &Command{ @@ -160,24 +162,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { ctx := context.Background() message := resp.EventNotification - if message.OldEntry == nil && message.NewEntry == nil { + if filer_pb.IsEmpty(resp) { return nil - } - if message.OldEntry == nil && message.NewEntry != nil { + } else if filer_pb.IsCreate(resp) { println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) return store.InsertEntry(ctx, entry) - } - if message.OldEntry != nil && message.NewEntry == nil { + } else if filer_pb.IsDelete(resp) { println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)) - } - if message.OldEntry != nil && message.NewEntry != nil { - if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { - println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) - entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) - return store.UpdateEntry(ctx, entry) - } + } else if filer_pb.IsUpdate(resp) { + println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) + entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) + return store.UpdateEntry(ctx, entry) + } else { + // renaming println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil { return err @@ -195,8 +194,8 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { return metaBackup.setOffset(lastTime) }) - return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup", - *metaBackup.filerDirectory, startTime.UnixNano(), 0, processEventFnWithOffset, false) + return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, + *metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError) } @@ -222,9 +221,9 @@ func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) err var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{}) -func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { return fn(client) }) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index 28c0db99b..66a87c3d9 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -1,12 +1,9 @@ package command import ( - "context" "fmt" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/golang/protobuf/jsonpb" - jsoniter "github.com/json-iterator/go" - "github.com/olivere/elastic/v7" "os" "path/filepath" "strings" @@ -28,8 +25,11 @@ var cmdFilerMetaTail = &Command{ weed filer.meta.tail -timeAgo=30h | grep truncate weed filer.meta.tail -timeAgo=30h | jq . + weed filer.meta.tail -timeAgo=30h -untilTimeAgo=20h | jq . weed filer.meta.tail -timeAgo=30h | jq .eventNotification.newEntry.name + weed filer.meta.tail -timeAgo=30h -es=http://: -es.index=seaweedfs + `, } @@ -37,6 +37,7 @@ var ( tailFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port") tailTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or common prefix for the folders or files on filer") tailStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Âĩs\"), \"ms\", \"s\", \"m\", \"h\"") + tailStop = cmdFilerMetaTail.Flag.Duration("untilTimeAgo", 0, "read until this time ago. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Âĩs\"), \"ms\", \"s\", \"m\", \"h\"") tailPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") esServers = cmdFilerMetaTail.Flag.String("es", "", "comma-separated elastic servers http://") esIndex = cmdFilerMetaTail.Flag.String("es.index", "seaweedfs", "ES index name") @@ -46,6 +47,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool { util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + clientId := util.RandomInt32() var filterFunc func(dir, fname string) bool if *tailPattern != "" { @@ -71,12 +73,12 @@ func runFilerMetaTail(cmd *Command, args []string) bool { } shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool { + if filer_pb.IsEmpty(resp) { + return false + } if filterFunc == nil { return true } - if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil { - return false - } if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) { return true } @@ -103,9 +105,13 @@ func runFilerMetaTail(cmd *Command, args []string) bool { } } - tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail", - *tailTarget, time.Now().Add(-*tailStart).UnixNano(), 0, - func(resp *filer_pb.SubscribeMetadataResponse) error { + var untilTsNs int64 + if *tailStop != 0 { + untilTsNs = time.Now().Add(-*tailStop).UnixNano() + } + + tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, *tailTarget, nil, + time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error { if !shouldPrint(resp) { return nil } @@ -113,7 +119,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return err } return nil - }, false) + }, pb.TrivialOnError) if tailErr != nil { fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) @@ -121,72 +127,3 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return true } - -type EsDocument struct { - Dir string `json:"dir,omitempty"` - Name string `json:"name,omitempty"` - IsDirectory bool `json:"isDir,omitempty"` - Size uint64 `json:"size,omitempty"` - Uid uint32 `json:"uid,omitempty"` - Gid uint32 `json:"gid,omitempty"` - UserName string `json:"userName,omitempty"` - Collection string `json:"collection,omitempty"` - Crtime int64 `json:"crtime,omitempty"` - Mtime int64 `json:"mtime,omitempty"` - Mime string `json:"mime,omitempty"` -} - -func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { - entry := event.NewEntry - dir, name := event.NewParentPath, entry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - esEntry := &EsDocument{ - Dir: dir, - Name: name, - IsDirectory: entry.IsDirectory, - Size: entry.Attributes.FileSize, - Uid: entry.Attributes.Uid, - Gid: entry.Attributes.Gid, - UserName: entry.Attributes.UserName, - Collection: entry.Attributes.Collection, - Crtime: entry.Attributes.Crtime, - Mtime: entry.Attributes.Mtime, - Mime: entry.Attributes.Mime, - } - return esEntry, id -} - -func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { - options := []elastic.ClientOptionFunc{} - options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) - options = append(options, elastic.SetSniff(false)) - options = append(options, elastic.SetHealthcheck(false)) - client, err := elastic.NewClient(options...) - if err != nil { - return nil, err - } - return func(resp *filer_pb.SubscribeMetadataResponse) error { - event := resp.EventNotification - if event.OldEntry != nil && - (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { - // delete or not update the same file - dir, name := resp.Directory, event.OldEntry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - println("delete", id) - _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) - return err - } - if event.NewEntry != nil { - // add a new file or update the same file - esEntry, id := toEsEntry(event) - value, err := jsoniter.Marshal(esEntry) - if err != nil { - return err - } - println(string(value)) - _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) - return err - } - return nil - }, nil -} diff --git a/weed/command/filer_meta_tail_elastic.go b/weed/command/filer_meta_tail_elastic.go new file mode 100644 index 000000000..5776c4f97 --- /dev/null +++ b/weed/command/filer_meta_tail_elastic.go @@ -0,0 +1,80 @@ +//go:build elastic +// +build elastic + +package command + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + jsoniter "github.com/json-iterator/go" + elastic "github.com/olivere/elastic/v7" + "strings" +) + +type EsDocument struct { + Dir string `json:"dir,omitempty"` + Name string `json:"name,omitempty"` + IsDirectory bool `json:"isDir,omitempty"` + Size uint64 `json:"size,omitempty"` + Uid uint32 `json:"uid,omitempty"` + Gid uint32 `json:"gid,omitempty"` + UserName string `json:"userName,omitempty"` + Crtime int64 `json:"crtime,omitempty"` + Mtime int64 `json:"mtime,omitempty"` + Mime string `json:"mime,omitempty"` +} + +func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { + entry := event.NewEntry + dir, name := event.NewParentPath, entry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + esEntry := &EsDocument{ + Dir: dir, + Name: name, + IsDirectory: entry.IsDirectory, + Size: entry.Attributes.FileSize, + Uid: entry.Attributes.Uid, + Gid: entry.Attributes.Gid, + UserName: entry.Attributes.UserName, + Crtime: entry.Attributes.Crtime, + Mtime: entry.Attributes.Mtime, + Mime: entry.Attributes.Mime, + } + return esEntry, id +} + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + options := []elastic.ClientOptionFunc{} + options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) + options = append(options, elastic.SetSniff(false)) + options = append(options, elastic.SetHealthcheck(false)) + client, err := elastic.NewClient(options...) + if err != nil { + return nil, err + } + return func(resp *filer_pb.SubscribeMetadataResponse) error { + event := resp.EventNotification + if event.OldEntry != nil && + (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { + // delete or not update the same file + dir, name := resp.Directory, event.OldEntry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + println("delete", id) + _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) + return err + } + if event.NewEntry != nil { + // add a new file or update the same file + esEntry, id := toEsEntry(event) + value, err := jsoniter.Marshal(esEntry) + if err != nil { + return err + } + println(string(value)) + _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) + return err + } + return nil + }, nil +} diff --git a/weed/command/filer_meta_tail_non_elastic.go b/weed/command/filer_meta_tail_non_elastic.go new file mode 100644 index 000000000..f78f3ee09 --- /dev/null +++ b/weed/command/filer_meta_tail_non_elastic.go @@ -0,0 +1,14 @@ +//go:build !elastic +// +build !elastic + +package command + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + return func(resp *filer_pb.SubscribeMetadataResponse) error { + return nil + }, nil +} diff --git a/weed/command/filer_remote_gateway.go b/weed/command/filer_remote_gateway.go new file mode 100644 index 000000000..33454f378 --- /dev/null +++ b/weed/command/filer_remote_gateway.go @@ -0,0 +1,119 @@ +package command + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/replication/source" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" + "os" + "time" +) + +type RemoteGatewayOptions struct { + filerAddress *string + grpcDialOption grpc.DialOption + readChunkFromFiler *bool + timeAgo *time.Duration + createBucketAt *string + createBucketRandomSuffix *bool + include *string + exclude *string + + mappings *remote_pb.RemoteStorageMapping + remoteConfs map[string]*remote_pb.RemoteConf + bucketsDir string + clientId int32 +} + +var _ = filer_pb.FilerClient(&RemoteGatewayOptions{}) + +func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { + return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return fn(client) + }) +} +func (option *RemoteGatewayOptions) AdjustedUrl(location *filer_pb.Location) string { + return location.Url +} + +var ( + remoteGatewayOptions RemoteGatewayOptions +) + +func init() { + cmdFilerRemoteGateway.Run = runFilerRemoteGateway // break init cycle + remoteGatewayOptions.filerAddress = cmdFilerRemoteGateway.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster") + remoteGatewayOptions.createBucketAt = cmdFilerRemoteGateway.Flag.String("createBucketAt", "", "one remote storage name to create new buckets in") + remoteGatewayOptions.createBucketRandomSuffix = cmdFilerRemoteGateway.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts") + remoteGatewayOptions.readChunkFromFiler = cmdFilerRemoteGateway.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers") + remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Âĩs\"), \"ms\", \"s\", \"m\", \"h\"") + remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*") + remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*") + remoteGatewayOptions.clientId = util.RandomInt32() +} + +var cmdFilerRemoteGateway = &Command{ + UsageLine: "filer.remote.gateway", + Short: "resumable continuously write back bucket creation, deletion, and other local updates to remote object store", + Long: `resumable continuously write back bucket creation, deletion, and other local updates to remote object store + + filer.remote.gateway listens on filer local buckets update events. + If any bucket is created, deleted, or updated, it will mirror the changes to remote object store. + + weed filer.remote.sync -createBucketAt=cloud1 + +`, +} + +func runFilerRemoteGateway(cmd *Command, args []string) bool { + + util.LoadConfiguration("security", false) + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + remoteGatewayOptions.grpcDialOption = grpcDialOption + + filerAddress := pb.ServerAddress(*remoteGatewayOptions.filerAddress) + + filerSource := &source.FilerSource{} + filerSource.DoInitialize( + filerAddress.ToHttpAddress(), + filerAddress.ToGrpcAddress(), + "/", // does not matter + *remoteGatewayOptions.readChunkFromFiler, + ) + + remoteGatewayOptions.bucketsDir = "/buckets" + // check buckets again + remoteGatewayOptions.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error { + resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return err + } + remoteGatewayOptions.bucketsDir = resp.DirBuckets + return nil + }) + + // read filer remote storage mount mappings + if detectErr := remoteGatewayOptions.collectRemoteStorageConf(); detectErr != nil { + fmt.Fprintf(os.Stderr, "read mount info: %v\n", detectErr) + return true + } + + // synchronize /buckets folder + fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir) + util.RetryForever("filer.remote.sync buckets", func() error { + return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource) + }, func(err error) bool { + if err != nil { + glog.Errorf("synchronize %s: %v", remoteGatewayOptions.bucketsDir, err) + } + return true + }) + return true + +} diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go new file mode 100644 index 000000000..c3ff756db --- /dev/null +++ b/weed/command/filer_remote_gateway_buckets.go @@ -0,0 +1,400 @@ +package command + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/replication/source" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/golang/protobuf/proto" + "math" + "math/rand" + "path/filepath" + "strings" + "time" +) + +func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error { + + // read filer remote storage mount mappings + if detectErr := option.collectRemoteStorageConf(); detectErr != nil { + return fmt.Errorf("read mount info: %v", detectErr) + } + + eachEntryFunc, err := option.makeBucketedEventProcessor(filerSource) + if err != nil { + return err + } + + processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error { + lastTime := time.Unix(0, lastTsNs) + glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3)) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs) + }) + + lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo) + + return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, + option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError) +} + +func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) { + + handleCreateBucket := func(entry *filer_pb.Entry) error { + if !entry.IsDirectory { + return nil + } + if entry.RemoteEntry != nil { + // this directory is imported from "remote.mount.buckets" or "remote.mount" + return nil + } + if option.mappings.PrimaryBucketStorageName != "" && *option.createBucketAt == "" { + *option.createBucketAt = option.mappings.PrimaryBucketStorageName + glog.V(0).Infof("%s is set as the primary remote storage", *option.createBucketAt) + } + if len(option.mappings.Mappings) == 1 && *option.createBucketAt == "" { + for k := range option.mappings.Mappings { + *option.createBucketAt = k + glog.V(0).Infof("%s is set as the only remote storage", *option.createBucketAt) + } + } + if *option.createBucketAt == "" { + return nil + } + remoteConf, found := option.remoteConfs[*option.createBucketAt] + if !found { + return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt) + } + + client, err := remote_storage.GetRemoteStorage(remoteConf) + if err != nil { + return err + } + + bucketName := strings.ToLower(entry.Name) + if *option.include != "" { + if ok, _ := filepath.Match(*option.include, entry.Name); !ok { + return nil + } + } + if *option.exclude != "" { + if ok, _ := filepath.Match(*option.exclude, entry.Name); ok { + return nil + } + } + + bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name) + remoteLocation, found := option.mappings.Mappings[string(bucketPath)] + if !found { + if *option.createBucketRandomSuffix { + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + if len(bucketName)+5 > 63 { + bucketName = bucketName[:58] + } + bucketName = fmt.Sprintf("%s-%04d", bucketName, rand.Uint32()%10000) + } + remoteLocation = &remote_pb.RemoteStorageLocation{ + Name: *option.createBucketAt, + Bucket: bucketName, + Path: "/", + } + // need to add new mapping here before getting updates from metadata tailing + option.mappings.Mappings[string(bucketPath)] = remoteLocation + } else { + bucketName = remoteLocation.Bucket + } + + glog.V(0).Infof("create bucket %s", bucketName) + if err := client.CreateBucket(bucketName); err != nil { + return fmt.Errorf("create bucket %s in %s: %v", bucketName, remoteConf.Name, err) + } + + return filer.InsertMountMapping(option, string(bucketPath), remoteLocation) + + } + handleDeleteBucket := func(entry *filer_pb.Entry) error { + if !entry.IsDirectory { + return nil + } + + client, remoteStorageMountLocation, err := option.findRemoteStorageClient(entry.Name) + if err != nil { + return fmt.Errorf("findRemoteStorageClient %s: %v", entry.Name, err) + } + + glog.V(0).Infof("delete remote bucket %s", remoteStorageMountLocation.Bucket) + if err := client.DeleteBucket(remoteStorageMountLocation.Bucket); err != nil { + return fmt.Errorf("delete remote bucket %s: %v", remoteStorageMountLocation.Bucket, err) + } + + bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name) + + return filer.DeleteMountMapping(option, string(bucketPath)) + } + + handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + if message.NewEntry != nil { + // update + if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE { + newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content) + if readErr != nil { + return fmt.Errorf("unmarshal mappings: %v", readErr) + } + option.mappings = newMappings + } + if strings.HasSuffix(message.NewEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) { + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil { + return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err) + } + option.remoteConfs[conf.Name] = conf + } + } else if message.OldEntry != nil { + // deletion + if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) { + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil { + return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.OldEntry.Name, err) + } + delete(option.remoteConfs, conf.Name) + } + } + + return nil + } + + eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) { + return handleEtcRemoteChanges(resp) + } + + if filer_pb.IsEmpty(resp) { + return nil + } + if filer_pb.IsCreate(resp) { + if message.NewParentPath == option.bucketsDir { + return handleCreateBucket(message.NewEntry) + } + if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/.uploads/") { + return nil + } + if !filer.HasData(message.NewEntry) { + return nil + } + bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(message.NewParentPath) + if !ok { + return nil + } + client, err := remote_storage.GetRemoteStorage(remoteStorage) + if err != nil { + return err + } + glog.V(2).Infof("create: %+v", resp) + if !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping creating: %+v", resp) + return nil + } + dest := toRemoteStorageLocation(bucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) + if message.NewEntry.IsDirectory { + glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) + return client.WriteDirectory(dest, message.NewEntry) + } + glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest)) + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest) + if writeErr != nil { + return writeErr + } + return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) + } + if filer_pb.IsDelete(resp) { + if resp.Directory == option.bucketsDir { + return handleDeleteBucket(message.OldEntry) + } + bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory) + if !ok { + return nil + } + client, err := remote_storage.GetRemoteStorage(remoteStorage) + if err != nil { + return err + } + glog.V(2).Infof("delete: %+v", resp) + dest := toRemoteStorageLocation(bucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) + if message.OldEntry.IsDirectory { + glog.V(0).Infof("rmdir %s", remote_storage.FormatLocation(dest)) + return client.RemoveDirectory(dest) + } + glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest)) + return client.DeleteFile(dest) + } + if message.OldEntry != nil && message.NewEntry != nil { + if resp.Directory == option.bucketsDir { + if message.NewParentPath == option.bucketsDir { + if message.OldEntry.Name == message.NewEntry.Name { + return nil + } + if err := handleCreateBucket(message.NewEntry); err != nil { + return err + } + if err := handleDeleteBucket(message.OldEntry); err != nil { + return err + } + } + } + oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory) + newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath) + if oldOk && newOk { + if !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping updating: %+v", resp) + return nil + } + client, err := remote_storage.GetRemoteStorage(oldRemoteStorage) + if err != nil { + return err + } + if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { + // update the same entry + if message.NewEntry.IsDirectory { + // update directory property + return nil + } + if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) { + glog.V(2).Infof("update meta: %+v", resp) + oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation) + return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry) + } else { + newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation) + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest) + if writeErr != nil { + return writeErr + } + return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) + } + } + } + + // the following is entry rename + if oldOk { + client, err := remote_storage.GetRemoteStorage(oldRemoteStorage) + if err != nil { + return err + } + oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation) + if message.OldEntry.IsDirectory { + return client.RemoveDirectory(oldDest) + } + glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest)) + if err := client.DeleteFile(oldDest); err != nil { + return err + } + } + if newOk { + if !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping updating: %+v", resp) + return nil + } + client, err := remote_storage.GetRemoteStorage(newRemoteStorage) + if err != nil { + return err + } + newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation) + if message.NewEntry.IsDirectory { + return client.WriteDirectory(newDest, message.NewEntry) + } + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest) + if writeErr != nil { + return writeErr + } + return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) + } + } + + return nil + } + return eachEntryFunc, nil +} + +func (option *RemoteGatewayOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) { + bucket := util.FullPath(option.bucketsDir).Child(bucketName) + + var isMounted bool + remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)] + if !isMounted { + return nil, remoteStorageMountLocation, fmt.Errorf("%s is not mounted", bucket) + } + remoteConf, hasClient := option.remoteConfs[remoteStorageMountLocation.Name] + if !hasClient { + return nil, remoteStorageMountLocation, fmt.Errorf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation) + } + + client, err = remote_storage.GetRemoteStorage(remoteConf) + if err != nil { + return nil, remoteStorageMountLocation, err + } + return client, remoteStorageMountLocation, nil +} + +func (option *RemoteGatewayOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) { + bucket, ok = extractBucketPath(option.bucketsDir, actualDir) + if !ok { + return "", nil, nil, false + } + var isMounted bool + remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)] + if !isMounted { + glog.Warningf("%s is not mounted", bucket) + return "", nil, nil, false + } + var hasClient bool + remoteConf, hasClient = option.remoteConfs[remoteStorageMountLocation.Name] + if !hasClient { + glog.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation) + return "", nil, nil, false + } + return bucket, remoteStorageMountLocation, remoteConf, true +} + +func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) { + if !strings.HasPrefix(dir, bucketsDir+"/") { + return "", false + } + parts := strings.SplitN(dir[len(bucketsDir)+1:], "/", 2) + return util.FullPath(bucketsDir).Child(parts[0]), true +} + +func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) { + + if mappings, err := filer.ReadMountMappings(option.grpcDialOption, pb.ServerAddress(*option.filerAddress)); err != nil { + return err + } else { + option.mappings = mappings + } + + option.remoteConfs = make(map[string]*remote_pb.RemoteConf) + var lastConfName string + err = filer_pb.List(option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error { + if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) { + return nil + } + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(entry.Content, conf); err != nil { + return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err) + } + option.remoteConfs[conf.Name] = conf + lastConfName = conf.Name + return nil + }, "", false, math.MaxUint32) + + if option.mappings.PrimaryBucketStorageName == "" && len(option.remoteConfs) == 1 { + glog.V(0).Infof("%s is set to the default remote storage", lastConfName) + option.mappings.PrimaryBucketStorageName = lastConfName + } + + return +} diff --git a/weed/command/filer_remote_sync.go b/weed/command/filer_remote_sync.go index 8d2719660..d6ccf7b79 100644 --- a/weed/command/filer_remote_sync.go +++ b/weed/command/filer_remote_sync.go @@ -1,14 +1,10 @@ package command import ( - "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" - "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" @@ -20,19 +16,15 @@ type RemoteSyncOptions struct { filerAddress *string grpcDialOption grpc.DialOption readChunkFromFiler *bool - debug *bool timeAgo *time.Duration dir *string + clientId int32 } -const ( - RemoteSyncKeyPrefix = "remote.sync." -) - var _ = filer_pb.FilerClient(&RemoteSyncOptions{}) -func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithFilerClient(*option.filerAddress, option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { + return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { return fn(client) }) } @@ -47,20 +39,28 @@ var ( func init() { cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster") - remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "/", "a mounted directory on filer") + remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer") remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers") - remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files") - remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Âĩs\"), \"ms\", \"s\", \"m\", \"h\"") + remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now, skipping previous metadata changes. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"Âĩs\"), \"ms\", \"s\", \"m\", \"h\"") + remoteSyncOptions.clientId = util.RandomInt32() } var cmdFilerRemoteSynchronize = &Command{ - UsageLine: "filer.remote.sync -filer=: -dir=/mount/s3_on_cloud", - Short: "resumable continuously write back updates to remote storage if the directory is mounted to the remote storage", - Long: `resumable continuously write back updates to remote storage if the directory is mounted to the remote storage + UsageLine: "filer.remote.sync", + Short: "resumable continuously write back updates to remote storage", + Long: `resumable continuously write back updates to remote storage filer.remote.sync listens on filer update events. If any mounted remote file is updated, it will fetch the updated content, and write to the remote storage. + + weed filer.remote.sync -dir=/mount/s3_on_cloud + + The metadata sync starting time is determined with the following priority order: + 1. specified by timeAgo + 2. last sync timestamp for this directory + 3. directory creation time + `, } @@ -71,176 +71,29 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool { remoteSyncOptions.grpcDialOption = grpcDialOption dir := *remoteSyncOptions.dir - filerAddress := *remoteSyncOptions.filerAddress + filerAddress := pb.ServerAddress(*remoteSyncOptions.filerAddress) filerSource := &source.FilerSource{} filerSource.DoInitialize( - filerAddress, - pb.ServerToGrpcAddress(filerAddress), + filerAddress.ToHttpAddress(), + filerAddress.ToGrpcAddress(), "/", // does not matter *remoteSyncOptions.readChunkFromFiler, ) - fmt.Printf("synchronize %s to remote storage...\n", dir) - util.RetryForever("filer.remote.sync "+dir, func() error { - return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir) - }, func(err error) bool { - if err != nil { - glog.Errorf("synchronize %s: %v", dir, err) - } - return true - }) - - return true -} - -func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error { - - // read filer remote storage mount mappings - _, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, *option.filerAddress, mountedDir) - if detectErr != nil { - return fmt.Errorf("read mount info: %v", detectErr) - } - - dirHash := util.HashStringToLong(mountedDir) - - // 1. specified by timeAgo - // 2. last offset timestamp for this directory - // 3. directory creation time - var lastOffsetTs time.Time - if *option.timeAgo == 0 { - mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir)) - if err != nil { - return fmt.Errorf("lookup %s: %v", mountedDir, err) - } - - lastOffsetTsNs, err := getOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash)) - if mountedDirEntry != nil { - if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 { - lastOffsetTs = time.Unix(0, lastOffsetTsNs) - glog.V(0).Infof("resume from %v", lastOffsetTs) - } else { - lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0) - } - } else { - lastOffsetTs = time.Now() - } - } else { - lastOffsetTs = time.Now().Add(-*option.timeAgo) - } - - client, err := remote_storage.GetRemoteStorage(remoteStorage) - if err != nil { - return err - } - - eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { - message := resp.EventNotification - if message.OldEntry == nil && message.NewEntry == nil { - return nil - } - if message.OldEntry == nil && message.NewEntry != nil { - if !filer.HasData(message.NewEntry) { - return nil - } - glog.V(2).Infof("create: %+v", resp) - if !shouldSendToRemote(message.NewEntry) { - glog.V(2).Infof("skipping creating: %+v", resp) - return nil - } - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) - if message.NewEntry.IsDirectory { - glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) - return client.WriteDirectory(dest, message.NewEntry) - } - glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest)) - reader := filer.NewFileReader(filerSource, message.NewEntry) - remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader) - if writeErr != nil { - return writeErr - } - return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) - } - if message.OldEntry != nil && message.NewEntry == nil { - glog.V(2).Infof("delete: %+v", resp) - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) - if message.OldEntry.IsDirectory { - glog.V(0).Infof("rmdir %s", remote_storage.FormatLocation(dest)) - return client.RemoveDirectory(dest) + if dir != "" { + fmt.Printf("synchronize %s to remote storage...\n", dir) + util.RetryForever("filer.remote.sync "+dir, func() error { + return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir) + }, func(err error) bool { + if err != nil { + glog.Errorf("synchronize %s: %v", dir, err) } - glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest)) - return client.DeleteFile(dest) - } - if message.OldEntry != nil && message.NewEntry != nil { - oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) - if !shouldSendToRemote(message.NewEntry) { - glog.V(2).Infof("skipping updating: %+v", resp) - return nil - } - if message.NewEntry.IsDirectory { - return client.WriteDirectory(dest, message.NewEntry) - } - if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { - if filer.IsSameData(message.OldEntry, message.NewEntry) { - glog.V(2).Infof("update meta: %+v", resp) - return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry) - } - } - glog.V(2).Infof("update: %+v", resp) - glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest)) - if err := client.DeleteFile(oldDest); err != nil { - return err - } - reader := filer.NewFileReader(filerSource, message.NewEntry) - glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest)) - remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader) - if writeErr != nil { - return writeErr - } - return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) - } - - return nil - } - - processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error { - lastTime := time.Unix(0, lastTsNs) - glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3)) - return setOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash), lastTsNs) - }) - - return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, - "filer.remote.sync", mountedDir, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false) -} - -func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation { - source := string(sourcePath[len(mountDir):]) - dest := util.FullPath(remoteMountLocation.Path).Child(source) - return &remote_pb.RemoteStorageLocation{ - Name: remoteMountLocation.Name, - Bucket: remoteMountLocation.Bucket, - Path: string(dest), - } -} - -func shouldSendToRemote(entry *filer_pb.Entry) bool { - if entry.RemoteEntry == nil { - return true - } - if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime { + return true + }) return true } - return false -} -func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error { - entry.RemoteEntry = remoteEntry - return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{ - Directory: dir, - Entry: entry, - }) - return err - }) + return true + } diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go new file mode 100644 index 000000000..5fc20be9a --- /dev/null +++ b/weed/command/filer_remote_sync_dir.go @@ -0,0 +1,237 @@ +package command + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/replication/source" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" +) + +func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error { + + // read filer remote storage mount mappings + _, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir) + if detectErr != nil { + return fmt.Errorf("read mount info: %v", detectErr) + } + + eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource) + if err != nil { + return err + } + + processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error { + lastTime := time.Unix(0, lastTsNs) + glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3)) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs) + }) + + lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo) + + return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, + mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError) +} + +func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) { + client, err := remote_storage.GetRemoteStorage(remoteStorage) + if err != nil { + return nil, err + } + + handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + if message.NewEntry == nil { + return nil + } + if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE { + mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content) + if readErr != nil { + return fmt.Errorf("unmarshal mappings: %v", readErr) + } + if remoteLoc, found := mappings.Mappings[mountedDir]; found { + if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path { + glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc) + } + } else { + glog.V(0).Infof("unmounted %s exiting ...", mountedDir) + os.Exit(0) + } + } + if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX { + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil { + return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err) + } + remoteStorage = conf + if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil { + client = newClient + } else { + return err + } + } + + return nil + } + + eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) { + return handleEtcRemoteChanges(resp) + } + + if filer_pb.IsEmpty(resp) { + return nil + } + if filer_pb.IsCreate(resp) { + if !filer.HasData(message.NewEntry) { + return nil + } + glog.V(2).Infof("create: %+v", resp) + if !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping creating: %+v", resp) + return nil + } + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) + if message.NewEntry.IsDirectory { + glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) + return client.WriteDirectory(dest, message.NewEntry) + } + glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest)) + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest) + if writeErr != nil { + return writeErr + } + return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) + } + if filer_pb.IsDelete(resp) { + glog.V(2).Infof("delete: %+v", resp) + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) + if message.OldEntry.IsDirectory { + glog.V(0).Infof("rmdir %s", remote_storage.FormatLocation(dest)) + return client.RemoveDirectory(dest) + } + glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest)) + return client.DeleteFile(dest) + } + if message.OldEntry != nil && message.NewEntry != nil { + oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) + if !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping updating: %+v", resp) + return nil + } + if message.NewEntry.IsDirectory { + return client.WriteDirectory(dest, message.NewEntry) + } + if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { + if filer.IsSameData(message.OldEntry, message.NewEntry) { + glog.V(2).Infof("update meta: %+v", resp) + return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry) + } + } + glog.V(2).Infof("update: %+v", resp) + glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest)) + if err := client.DeleteFile(oldDest); err != nil { + return err + } + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest) + if writeErr != nil { + return writeErr + } + return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry) + } + + return nil + } + return eachEntryFunc, nil +} + +func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) { + var writeErr error + err = util.Retry("writeFile", func() error { + reader := filer.NewFileReader(filerSource, newEntry) + glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest)) + remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader) + if writeErr != nil { + return writeErr + } + return nil + }) + if err != nil { + glog.Errorf("write to %s: %v", dest, err) + } + return +} + +func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time { + // 1. specified by timeAgo + // 2. last offset timestamp for this directory + // 3. directory creation time + var lastOffsetTs time.Time + if timeAgo == 0 { + mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir)) + if err != nil { + glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err) + return time.Now() + } + + lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir) + if mountedDirEntry != nil { + if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 { + lastOffsetTs = time.Unix(0, lastOffsetTsNs) + glog.V(0).Infof("resume from %v", lastOffsetTs) + } else { + lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0) + } + } else { + lastOffsetTs = time.Now() + } + } else { + lastOffsetTs = time.Now().Add(-timeAgo) + } + return lastOffsetTs +} + +func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation { + source := string(sourcePath[len(mountDir):]) + dest := util.FullPath(remoteMountLocation.Path).Child(source) + return &remote_pb.RemoteStorageLocation{ + Name: remoteMountLocation.Name, + Bucket: remoteMountLocation.Bucket, + Path: string(dest), + } +} + +func shouldSendToRemote(entry *filer_pb.Entry) bool { + if entry.RemoteEntry == nil { + return true + } + if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime { + return true + } + return false +} + +func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error { + remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano() + entry.RemoteEntry = remoteEntry + return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{ + Directory: dir, + Entry: entry, + }) + return err + }) +} diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 5440811dd..b7da1baf9 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -12,9 +12,11 @@ import ( "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" "github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/security" + statsCollect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/grace" "google.golang.org/grpc" + "os" "strings" "time" ) @@ -35,8 +37,12 @@ type SyncOptions struct { bDiskType *string aDebug *bool bDebug *bool + aFromTsMs *int64 + bFromTsMs *int64 aProxyByFiler *bool bProxyByFiler *bool + metricsHttpPort *int + clientId int32 } var ( @@ -64,8 +70,12 @@ func init() { syncOptions.bProxyByFiler = cmdFilerSynchronize.Flag.Bool("b.filerProxy", false, "read and write file chunks by filer B instead of volume servers") syncOptions.aDebug = cmdFilerSynchronize.Flag.Bool("a.debug", false, "debug mode to print out filer A received files") syncOptions.bDebug = cmdFilerSynchronize.Flag.Bool("b.debug", false, "debug mode to print out filer B received files") + syncOptions.aFromTsMs = cmdFilerSynchronize.Flag.Int64("a.fromTsMs", 0, "synchronization from timestamp on filer A. The unit is millisecond") + syncOptions.bFromTsMs = cmdFilerSynchronize.Flag.Int64("b.fromTsMs", 0, "synchronization from timestamp on filer B. The unit is millisecond") syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file") syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file") + syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port") + syncOptions.clientId = util.RandomInt32() } var cmdFilerSynchronize = &Command{ @@ -93,10 +103,37 @@ func runFilerSynchronize(cmd *Command, args []string) bool { grace.SetupProfiling(*syncCpuProfile, *syncMemProfile) + filerA := pb.ServerAddress(*syncOptions.filerA) + filerB := pb.ServerAddress(*syncOptions.filerB) + + // start filer.sync metrics server + go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort) + + // read a filer signature + aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA) + if aFilerErr != nil { + glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr) + return true + } + // read b filer signature + bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB) + if bFilerErr != nil { + glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr) + return true + } + go func() { + // a->b + // set synchronization start timestamp to offset + initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs) + if initOffsetError != nil { + glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError) + os.Exit(2) + } for { - err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, *syncOptions.filerB, - *syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bProxyByFiler, *syncOptions.bDiskType, *syncOptions.bDebug) + err := doSubscribeFilerMetaChanges(syncOptions.clientId, grpcDialOption, filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, filerB, + *syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bProxyByFiler, *syncOptions.bDiskType, + *syncOptions.bDebug, aFilerSignature, bFilerSignature) if err != nil { glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err) time.Sleep(1747 * time.Millisecond) @@ -105,10 +142,18 @@ func runFilerSynchronize(cmd *Command, args []string) bool { }() if !*syncOptions.isActivePassive { + // b->a + // set synchronization start timestamp to offset + initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs) + if initOffsetError != nil { + glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError) + os.Exit(2) + } go func() { for { - err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, *syncOptions.filerA, - *syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aProxyByFiler, *syncOptions.aDiskType, *syncOptions.aDebug) + err := doSubscribeFilerMetaChanges(syncOptions.clientId, grpcDialOption, filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, filerA, + *syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aProxyByFiler, *syncOptions.aDiskType, + *syncOptions.aDebug, bFilerSignature, aFilerSignature) if err != nil { glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err) time.Sleep(2147 * time.Millisecond) @@ -122,23 +167,28 @@ func runFilerSynchronize(cmd *Command, args []string) bool { return true } -func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, sourcePath string, sourceReadChunkFromFiler bool, targetFiler, targetPath string, - replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool) error { - - // read source filer signature - sourceFilerSignature, sourceErr := replication.ReadFilerSignature(grpcDialOption, sourceFiler) - if sourceErr != nil { - return sourceErr +// initOffsetFromTsMs Initialize offset +func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, sourceFilerSignature int32, fromTsMs int64) error { + if fromTsMs <= 0 { + return nil } - // read target filer signature - targetFilerSignature, targetErr := replication.ReadFilerSignature(grpcDialOption, targetFiler) - if targetErr != nil { - return targetErr + // convert to nanosecond + fromTsNs := fromTsMs * 1000_000 + // If not successful, exit the program. + setOffsetErr := setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, fromTsNs) + if setOffsetErr != nil { + return setOffsetErr } + glog.Infof("setOffset from timestamp ms success! start offset: %d from %s to %s", fromTsNs, *syncOptions.filerA, *syncOptions.filerB) + return nil +} + +func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string, + replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, sourceFilerSignature int32, targetFilerSignature int32) error { // if first time, start from now // if has previously synced, resume from that point of time - sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature) + sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature) if err != nil { return err } @@ -147,9 +197,9 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so // create filer sink filerSource := &source.FilerSource{} - filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, sourceReadChunkFromFiler) + filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler) filerSink := &filersink.FilerSink{} - filerSink.DoInitialize(targetFiler, pb.ServerToGrpcAddress(targetFiler), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler) + filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler) filerSink.SetSourceFiler(filerSource) persistEventFn := genProcessFunction(sourcePath, targetPath, filerSink, debug) @@ -165,13 +215,19 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so return persistEventFn(resp) } + var lastLogTsNs = time.Now().Nanosecond() + var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler)) processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error { - glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3)) - return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs) + now := time.Now().Nanosecond() + glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) + lastLogTsNs = now + // collect synchronous offset + statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(lastTsNs)) + return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs) }) - return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler, - sourcePath, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false) + return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, + sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError) } @@ -179,9 +235,19 @@ const ( SyncKeyPrefix = "sync." ) -func getOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) { +// When each business is distinguished according to path, and offsets need to be maintained separately. +func getSignaturePrefixByPath(path string) string { + // compatible historical version + if path == "/" { + return SyncKeyPrefix + } else { + return SyncKeyPrefix + path + } +} + +func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) { - readErr = pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { syncKey := []byte(signaturePrefix + "____") util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) @@ -206,8 +272,8 @@ func getOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix str } -func setOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix string, signature int32, offsetTsNs int64) error { - return pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error { + return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { syncKey := []byte(signaturePrefix + "____") util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) @@ -255,16 +321,19 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl } // handle deletions - if message.OldEntry != nil && message.NewEntry == nil { + if filer_pb.IsDelete(resp) { if !strings.HasPrefix(string(sourceOldKey), sourcePath) { return nil } key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath) - return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures) + if !dataSink.IsIncremental() { + return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures) + } + return nil } // handle new entries - if message.OldEntry == nil && message.NewEntry != nil { + if filer_pb.IsCreate(resp) { if !strings.HasPrefix(string(sourceNewKey), sourcePath) { return nil } @@ -273,7 +342,7 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl } // this is something special? - if message.OldEntry == nil && message.NewEntry == nil { + if filer_pb.IsEmpty(resp) { return nil } diff --git a/weed/command/filer_sync_std.go b/weed/command/filer_sync_std.go index 63851eaf8..1f9b6fa14 100644 --- a/weed/command/filer_sync_std.go +++ b/weed/command/filer_sync_std.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package command diff --git a/weed/command/fix.go b/weed/command/fix.go index ae9a051b8..d19496a79 100644 --- a/weed/command/fix.go +++ b/weed/command/fix.go @@ -1,9 +1,12 @@ package command import ( + "fmt" + "io/fs" "os" "path" "strconv" + "strings" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage" @@ -19,17 +22,15 @@ func init() { } var cmdFix = &Command{ - UsageLine: "fix -dir=/tmp -volumeId=234", - Short: "run weed tool fix on index file if corrupted", - Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file. - + UsageLine: "fix [-volumeId=234] [-collection=bigData] /tmp", + Short: "run weed tool fix on files or whole folders to recreate index file(s) if corrupted", + Long: `Fix runs the SeaweedFS fix command on dat files or whole folders to re-create the index .idx file. `, } var ( - fixVolumePath = cmdFix.Flag.String("dir", ".", "data directory to store files") - fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name") - fixVolumeId = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") + fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed") + fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed") ) type VolumeFileScanner4Fix struct { @@ -59,26 +60,68 @@ func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64 } func runFix(cmd *Command, args []string) bool { - - if *fixVolumeId == -1 { - return false + for _, arg := range args { + basePath, f := path.Split(util.ResolvePath(arg)) + + files := []fs.DirEntry{} + if f == "" { + fileInfo, err := os.ReadDir(basePath) + if err != nil { + fmt.Println(err) + return false + } + files = fileInfo + } else { + fileInfo, err := os.Stat(basePath + f) + if err != nil { + fmt.Println(err) + return false + } + files = []fs.DirEntry{fs.FileInfoToDirEntry(fileInfo)} + } + + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".dat") { + continue + } + if *fixVolumeCollection != "" { + if !strings.HasPrefix(file.Name(), *fixVolumeCollection+"_") { + continue + } + } + baseFileName := file.Name()[:len(file.Name())-4] + collection, volumeIdStr := "", baseFileName + if sepIndex := strings.LastIndex(baseFileName, "_"); sepIndex > 0 { + collection = baseFileName[:sepIndex] + volumeIdStr = baseFileName[sepIndex+1:] + } + volumeId, parseErr := strconv.ParseInt(volumeIdStr, 10, 64) + if parseErr != nil { + fmt.Printf("Failed to parse volume id from %s: %v\n", baseFileName, parseErr) + return false + } + if *fixVolumeId != 0 && *fixVolumeId != volumeId { + continue + } + doFixOneVolume(basePath, baseFileName, collection, volumeId) + } } + return true +} - baseFileName := strconv.Itoa(*fixVolumeId) - if *fixVolumeCollection != "" { - baseFileName = *fixVolumeCollection + "_" + baseFileName - } - indexFileName := path.Join(util.ResolvePath(*fixVolumePath), baseFileName+".idx") +func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) { + + indexFileName := path.Join(basepath, baseFileName+".idx") nm := needle_map.NewMemDb() defer nm.Close() - vid := needle.VolumeId(*fixVolumeId) + vid := needle.VolumeId(volumeId) scanner := &VolumeFileScanner4Fix{ nm: nm, } - if err := storage.ScanVolumeFile(util.ResolvePath(*fixVolumePath), *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner); err != nil { + if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil { glog.Fatalf("scan .dat File: %v", err) os.Remove(indexFileName) } @@ -87,6 +130,4 @@ func runFix(cmd *Command, args []string) bool { glog.Fatalf("save to .idx File: %v", err) os.Remove(indexFileName) } - - return true } diff --git a/weed/command/iam.go b/weed/command/iam.go index ed4eea543..968d23095 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -22,6 +22,7 @@ var ( type IamOptions struct { filer *string masters *string + ip *string port *int } @@ -29,6 +30,7 @@ func init() { cmdIam.Run = runIam // break init cycle iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address") iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers") + iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address") iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port") } @@ -43,38 +45,35 @@ func runIam(cmd *Command, args []string) bool { } func (iamopt *IamOptions) startIamServer() bool { - filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer) - if err != nil { - glog.Fatal(err) - return false - } + filerAddress := pb.ServerAddress(*iamopt.filer) util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for { - err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) } glog.V(0).Infof("IAM read filer configuration: %s", resp) return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress) + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress()) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress) + glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress()) break } } + masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap() router := mux.NewRouter().SkipClean(true) _, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{ - Filer: *iamopt.filer, - Port: *iamopt.port, - FilerGrpcAddress: filerGrpcAddress, - GrpcDialOption: grpcDialOption, + Masters: masters, + Filer: filerAddress, + Port: *iamopt.port, + GrpcDialOption: grpcDialOption, }) glog.V(0).Info("NewIamApiServer created") if iamApiServer_err != nil { @@ -84,12 +83,19 @@ func (iamopt *IamOptions) startIamServer() bool { httpS := &http.Server{Handler: router} listenAddress := fmt.Sprintf(":%d", *iamopt.port) - iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second) + iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second) if err != nil { glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err) } glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port) + if iamApiLocalListener != nil { + go func() { + if err = httpS.Serve(iamApiLocalListener); err != nil { + glog.Errorf("IAM API Server Fail to serve: %v", err) + } + }() + } if err = httpS.Serve(iamApiListener); err != nil { glog.Fatalf("IAM API Server Fail to serve: %v", err) } diff --git a/weed/command/imports.go b/weed/command/imports.go index a2f59189f..afdbc5a10 100644 --- a/weed/command/imports.go +++ b/weed/command/imports.go @@ -5,7 +5,6 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure" _ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs" - _ "github.com/chrislusf/seaweedfs/weed/remote_storage/hdfs" _ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink" @@ -15,6 +14,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" @@ -29,6 +29,8 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/postgres2" _ "github.com/chrislusf/seaweedfs/weed/filer/redis" _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" + _ "github.com/chrislusf/seaweedfs/weed/filer/redis3" _ "github.com/chrislusf/seaweedfs/weed/filer/sqlite" _ "github.com/chrislusf/seaweedfs/weed/filer/tikv" + _ "github.com/chrislusf/seaweedfs/weed/filer/ydb" ) diff --git a/weed/command/master.go b/weed/command/master.go index 4eb43ee09..ab8466d47 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -1,23 +1,27 @@ package command import ( - "github.com/chrislusf/raft/protobuf" - "github.com/gorilla/mux" - "google.golang.org/grpc/reflection" + "fmt" + "golang.org/x/exp/slices" "net/http" "os" - "sort" - "strconv" + "path" "strings" "time" + "github.com/chrislusf/raft/protobuf" + stats_collect "github.com/chrislusf/seaweedfs/weed/stats" + "github.com/gorilla/mux" + "github.com/spf13/viper" + "google.golang.org/grpc/reflection" + "github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/server" + weed_server "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -28,6 +32,7 @@ var ( type MasterOptions struct { port *int + portGrpc *int ip *string ipBind *string metaFolder *string @@ -42,13 +47,19 @@ type MasterOptions struct { metricsAddress *string metricsIntervalSec *int raftResumeState *bool + metricsHttpPort *int + heartbeatInterval *time.Duration + electionTimeout *time.Duration + raftHashicorp *bool + raftBootstrap *bool } func init() { cmdMaster.Run = runMaster // break init cycle m.port = cmdMaster.Flag.Int("port", 9333, "http listen port") + m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port") m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master | address, also used as identifier") - m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to") + m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.") m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095") m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") @@ -60,7 +71,12 @@ func init() { m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.") m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address :") m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") + m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server") + m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") + m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers") + m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft") + m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster") } var cmdMaster = &Command{ @@ -103,6 +119,7 @@ func runMaster(cmd *Command, args []string) bool { glog.Fatalf("volumeSizeLimitMB should be smaller than 30000") } + go stats_collect.StartMetricsServer(*m.metricsHttpPort) startMaster(m, masterWhiteList) return true @@ -112,65 +129,139 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { backend.LoadConfiguration(util.GetViper()) - myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.peers) + if *masterOption.portGrpc == 0 { + *masterOption.portGrpc = 10000 + *masterOption.port + } + if *masterOption.ipBind == "" { + *masterOption.ipBind = *masterOption.ip + } + + myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers) + + masterPeers := make(map[string]pb.ServerAddress) + for _, peer := range peers { + masterPeers[string(peer)] = peer + } r := mux.NewRouter() - ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers) - listeningAddress := *masterOption.ipBind + ":" + strconv.Itoa(*masterOption.port) + ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers) + listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port) glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress) - masterListener, e := util.NewListener(listeningAddress, 0) + masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0) if e != nil { glog.Fatalf("Master startup error: %v", e) } + // start raftServer - raftServer, err := weed_server.NewRaftServer(security.LoadClientTLS(util.GetViper(), "grpc.master"), - peers, myMasterAddress, util.ResolvePath(*masterOption.metaFolder), ms.Topo, *masterOption.raftResumeState) - if raftServer == nil { - glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err) + metaDir := path.Join(*masterOption.metaFolder, fmt.Sprintf("m%d", *masterOption.port)) + raftServerOption := &weed_server.RaftServerOption{ + GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"), + Peers: masterPeers, + ServerAddr: myMasterAddress, + DataDir: util.ResolvePath(metaDir), + Topo: ms.Topo, + RaftResumeState: *masterOption.raftResumeState, + HeartbeatInterval: *masterOption.heartbeatInterval, + ElectionTimeout: *masterOption.electionTimeout, + RaftBootstrap: *m.raftBootstrap, + } + var raftServer *weed_server.RaftServer + var err error + if *m.raftHashicorp { + if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil { + glog.Fatalf("NewHashicorpRaftServer: %s", err) + } + } else { + raftServer, err = weed_server.NewRaftServer(raftServerOption) + if raftServer == nil { + glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err) + } } ms.SetRaftServer(raftServer) r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET") + if *m.raftHashicorp { + r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET") + } // starting grpc server - grpcPort := *masterOption.port + 10000 - grpcL, err := util.NewListener(*masterOption.ipBind+":"+strconv.Itoa(grpcPort), 0) + grpcPort := *masterOption.portGrpc + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0) if err != nil { glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) } grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master")) master_pb.RegisterSeaweedServer(grpcS, ms) - protobuf.RegisterRaftServer(grpcS, raftServer) + if *m.raftHashicorp { + raftServer.TransportManager.Register(grpcS) + } else { + protobuf.RegisterRaftServer(grpcS, raftServer) + } reflection.Register(grpcS) glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } go grpcS.Serve(grpcL) - go func() { - time.Sleep(1500 * time.Millisecond) - if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) { - if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" { - raftServer.DoJoinCommand() + timeSleep := 1500 * time.Millisecond + if !*m.raftHashicorp { + go func() { + time.Sleep(timeSleep) + if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) { + if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" { + raftServer.DoJoinCommand() + } } - } - }() + }() + } go ms.MasterClient.KeepConnectedToMaster() // start http server + var ( + clientCertFile, + certFile, + keyFile string + ) + useTLS := false + useMTLS := false + + if viper.GetString("https.master.key") != "" { + useTLS = true + certFile = viper.GetString("https.master.cert") + keyFile = viper.GetString("https.master.key") + } + + if viper.GetString("https.master.ca") != "" { + useMTLS = true + clientCertFile = viper.GetString("https.master.ca") + } + httpS := &http.Server{Handler: r} - go httpS.Serve(masterListener) + if masterLocalListner != nil { + go httpS.Serve(masterLocalListner) + } + + if useMTLS { + httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile) + } + + if useTLS { + go httpS.ServeTLS(masterListener, certFile, keyFile) + } else { + go httpS.Serve(masterListener) + } select {} } -func checkPeers(masterIp string, masterPort int, peers string) (masterAddress string, cleanedPeers []string) { +func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers string) (masterAddress pb.ServerAddress, cleanedPeers []pb.ServerAddress) { glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers) - masterAddress = masterIp + ":" + strconv.Itoa(masterPort) - if peers != "" { - cleanedPeers = strings.Split(peers, ",") - } + masterAddress = pb.NewServerAddress(masterIp, masterPort, masterGrpcPort) + cleanedPeers = pb.ServerAddresses(peers).ToAddresses() hasSelf := false for _, peer := range cleanedPeers { - if peer == masterAddress { + if peer.ToHttpAddress() == masterAddress.ToHttpAddress() { hasSelf = true break } @@ -180,13 +271,15 @@ func checkPeers(masterIp string, masterPort int, peers string) (masterAddress st cleanedPeers = append(cleanedPeers, masterAddress) } if len(cleanedPeers)%2 == 0 { - glog.Fatalf("Only odd number of masters are supported!") + glog.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers) } return } -func isTheFirstOne(self string, peers []string) bool { - sort.Strings(peers) +func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool { + slices.SortFunc(peers, func(a, b pb.ServerAddress) bool { + return strings.Compare(string(a), string(b)) < 0 + }) if len(peers) <= 0 { return true } @@ -194,9 +287,9 @@ func isTheFirstOne(self string, peers []string) bool { } func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption { + masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc) return &weed_server.MasterOption{ - Host: *m.ip, - Port: *m.port, + Master: masterAddress, MetaFolder: *m.metaFolder, VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB), VolumePreallocate: *m.volumePreallocate, diff --git a/weed/command/master_follower.go b/weed/command/master_follower.go index b628f7abf..ec7d2758f 100644 --- a/weed/command/master_follower.go +++ b/weed/command/master_follower.go @@ -3,19 +3,18 @@ package command import ( "context" "fmt" + "net/http" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/server" + weed_server "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" "github.com/gorilla/mux" "google.golang.org/grpc/reflection" - "net/http" - "strconv" - "strings" - "time" ) var ( @@ -25,7 +24,8 @@ var ( func init() { cmdMasterFollower.Run = runMasterFollower // break init cycle mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port") - mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to") + mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port") + mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.") mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095") mf.ip = aws.String(util.DetectedHostAddress()) @@ -46,13 +46,13 @@ var cmdMasterFollower = &Command{ Short: "start a master follower", Long: `start a master follower to provide volume=>location mapping service - The master follower does not participate in master election. + The master follower does not participate in master election. It just follow the existing masters, and listen for any volume location changes. In most cases, the master follower is not needed. In big data centers with thousands of volume servers. In theory, the master may have trouble to keep up with the write requests and read requests. - The master follower can relieve the master from from read requests, which only needs to + The master follower can relieve the master from from read requests, which only needs to lookup a fileId or volumeId. The master follower currently can handle fileId lookup requests: @@ -71,6 +71,10 @@ func runMasterFollower(cmd *Command, args []string) bool { util.LoadConfiguration("security", false) util.LoadConfiguration("master", false) + if *mf.portGrpc == 0 { + *mf.portGrpc = 10000 + *mf.port + } + startMasterFollower(mf) return true @@ -79,19 +83,15 @@ func runMasterFollower(cmd *Command, args []string) bool { func startMasterFollower(masterOptions MasterOptions) { // collect settings from main masters - masters := strings.Split(*mf.peers, ",") - masterGrpcAddresses, err := pb.ParseServersToGrpcAddresses(masters) - if err != nil { - glog.V(0).Infof("ParseFilerGrpcAddress: %v", err) - return - } + masters := pb.ServerAddresses(*mf.peers).ToAddressMap() + var err error grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master") for i := 0; i < 10; i++ { - err = pb.WithOneOfGrpcMasterClients(masterGrpcAddresses, grpcDialOption, func(client master_pb.SeaweedClient) error { + err = pb.WithOneOfGrpcMasterClients(false, masters, grpcDialOption, func(client master_pb.SeaweedClient) error { resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) if err != nil { - return fmt.Errorf("get master grpc address %v configuration: %v", masterGrpcAddresses, err) + return fmt.Errorf("get master grpc address %v configuration: %v", masters, err) } masterOptions.defaultReplication = &resp.DefaultReplication masterOptions.volumeSizeLimitMB = aws.Uint(uint(resp.VolumeSizeLimitMB)) @@ -99,31 +99,35 @@ func startMasterFollower(masterOptions MasterOptions) { return nil }) if err != nil { - glog.V(0).Infof("failed to talk to filer %v: %v", masterGrpcAddresses, err) + glog.V(0).Infof("failed to talk to filer %v: %v", masters, err) glog.V(0).Infof("wait for %d seconds ...", i+1) time.Sleep(time.Duration(i+1) * time.Second) } } if err != nil { - glog.Errorf("failed to talk to filer %v: %v", masterGrpcAddresses, err) + glog.Errorf("failed to talk to filer %v: %v", masters, err) return } option := masterOptions.toMasterOption(nil) option.IsFollower = true + if *masterOptions.ipBind == "" { + *masterOptions.ipBind = *masterOptions.ip + } + r := mux.NewRouter() ms := weed_server.NewMasterServer(r, option, masters) - listeningAddress := *masterOptions.ipBind + ":" + strconv.Itoa(*masterOptions.port) + listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port) glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress) - masterListener, e := util.NewListener(listeningAddress, 0) + masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0) if e != nil { glog.Fatalf("Master startup error: %v", e) } // starting grpc server - grpcPort := *masterOptions.port + 10000 - grpcL, err := util.NewListener(*masterOptions.ipBind+":"+strconv.Itoa(grpcPort), 0) + grpcPort := *masterOptions.portGrpc + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0) if err != nil { glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) } @@ -131,12 +135,18 @@ func startMasterFollower(masterOptions MasterOptions) { master_pb.RegisterSeaweedServer(grpcS, ms) reflection.Register(grpcS) glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } go grpcS.Serve(grpcL) go ms.MasterClient.KeepConnectedToMaster() // start http server httpS := &http.Server{Handler: r} + if masterLocalListner != nil { + go httpS.Serve(masterLocalListner) + } go httpS.Serve(masterListener) select {} diff --git a/weed/command/mount.go b/weed/command/mount.go index aec5fcc3c..0046ca03d 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -11,6 +11,7 @@ type MountOptions struct { dir *string dirAutoCreate *bool collection *string + collectionQuota *int replication *string diskType *string ttlSec *int @@ -26,6 +27,10 @@ type MountOptions struct { uidMap *string gidMap *string readOnly *bool + debug *bool + debugPort *int + localSocket *string + disableXAttr *bool } var ( @@ -42,13 +47,14 @@ func init() { mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") + mountOptions.collectionQuota = cmdMount.Flag.Int("collectionQuotaMB", 0, "quota for the collection") mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") - mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)") + mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system") mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") @@ -57,6 +63,10 @@ func init() { mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only") + mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") + mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging") + mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-.sock") + mountOptions.disableXAttr = cmdMount.Flag.Bool("disableXAttr", false, "disable xattr") mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file") mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file") @@ -76,7 +86,7 @@ var cmdMount = &Command{ This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on Linux, and OS X. - On OS X, it requires OSXFUSE (http://osxfuse.github.com/). + On OS X, it requires OSXFUSE (https://osxfuse.github.io/). `, } diff --git a/weed/command/mount_darwin.go b/weed/command/mount_darwin.go index f0a5581e7..05d6a1bc4 100644 --- a/weed/command/mount_darwin.go +++ b/weed/command/mount_darwin.go @@ -1,13 +1,5 @@ package command -import ( - "github.com/seaweedfs/fuse" -) - -func osSpecificMountOptions() []fuse.MountOption { - return []fuse.MountOption{} -} - func checkMountPointAvailable(dir string) bool { return true } diff --git a/weed/command/mount_freebsd.go b/weed/command/mount_freebsd.go deleted file mode 100644 index f0a5581e7..000000000 --- a/weed/command/mount_freebsd.go +++ /dev/null @@ -1,13 +0,0 @@ -package command - -import ( - "github.com/seaweedfs/fuse" -) - -func osSpecificMountOptions() []fuse.MountOption { - return []fuse.MountOption{} -} - -func checkMountPointAvailable(dir string) bool { - return true -} diff --git a/weed/command/mount_linux.go b/weed/command/mount_linux.go index 25c4f72cf..aebb14e61 100644 --- a/weed/command/mount_linux.go +++ b/weed/command/mount_linux.go @@ -6,8 +6,6 @@ import ( "io" "os" "strings" - - "github.com/seaweedfs/fuse" ) const ( @@ -137,10 +135,6 @@ func parseInfoFile(r io.Reader) ([]*Info, error) { return out, nil } -func osSpecificMountOptions() []fuse.MountOption { - return []fuse.MountOption{} -} - func checkMountPointAvailable(dir string) bool { mountPoint := dir if mountPoint != "/" && strings.HasSuffix(mountPoint, "/") { diff --git a/weed/command/mount_notsupported.go b/weed/command/mount_notsupported.go index f3c0de3d6..894c8e313 100644 --- a/weed/command/mount_notsupported.go +++ b/weed/command/mount_notsupported.go @@ -1,6 +1,5 @@ -// +build !linux -// +build !darwin -// +build !freebsd +//go:build !linux && !darwin +// +build !linux,!darwin package command diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index cdf340067..1aff3c5bb 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -1,38 +1,41 @@ -// +build linux darwin freebsd +//go:build linux || darwin +// +build linux darwin package command import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/mount/unmount" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/mount_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/hanwen/go-fuse/v2/fuse" + "google.golang.org/grpc/reflection" + "net" + "net/http" "os" "os/user" - "path" - "path/filepath" "runtime" "strconv" "strings" - "syscall" "time" - "github.com/chrislusf/seaweedfs/weed/storage/types" - - "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filesys" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/grace" ) func runMount(cmd *Command, args []string) bool { + if *mountOptions.debug { + go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil) + } + grace.SetupProfiling(*mountCpuProfile, *mountMemProfile) if *mountReadRetryTime < time.Second { *mountReadRetryTime = time.Second @@ -52,76 +55,67 @@ func runMount(cmd *Command, args []string) bool { return RunMount(&mountOptions, os.FileMode(umask)) } -func getParentInode(mountDir string) (uint64, error) { - parentDir := filepath.Clean(filepath.Join(mountDir, "..")) - fi, err := os.Stat(parentDir) - if err != nil { - return 0, err - } - - stat, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return 0, nil - } - - return stat.Ino, nil -} - func RunMount(option *MountOptions, umask os.FileMode) bool { - filers := strings.Split(*option.filer, ",") - // parse filer grpc address - filerGrpcAddresses, err := pb.ParseServersToGrpcAddresses(filers) - if err != nil { - glog.V(0).Infof("ParseFilerGrpcAddress: %v", err) - return true + // basic checks + chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB + if chunkSizeLimitMB <= 0 { + fmt.Printf("Please specify a reasonable buffer size.") + return false } + // try to connect to filer + filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses() util.LoadConfiguration("security", false) - // try to connect to filer, filerBucketsPath may be useful later grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var cipher bool + var err error for i := 0; i < 10; i++ { - err = pb.WithOneOfGrpcFilerClients(filerGrpcAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer grpc address %v configuration: %v", filerGrpcAddresses, err) + return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err) } cipher = resp.Cipher return nil }) if err != nil { - glog.V(0).Infof("failed to talk to filer %v: %v", filerGrpcAddresses, err) + glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err) glog.V(0).Infof("wait for %d seconds ...", i+1) time.Sleep(time.Duration(i+1) * time.Second) } } if err != nil { - glog.Errorf("failed to talk to filer %v: %v", filerGrpcAddresses, err) + glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err) return true } filerMountRootPath := *option.filerMountRootPath - dir := util.ResolvePath(*option.dir) - parentInode, err := getParentInode(dir) - if err != nil { - glog.Errorf("failed to retrieve inode for parent directory of %s: %v", dir, err) - return true - } - fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) + // clean up mount point + dir := util.ResolvePath(*option.dir) if dir == "" { fmt.Printf("Please specify the mount directory via \"-dir\"") return false } - chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB - if chunkSizeLimitMB <= 0 { - fmt.Printf("Please specify a reasonable buffer size.") - return false - } + unmount.Unmount(dir) - fuse.Unmount(dir) + // start on local unix socket + if *option.localSocket == "" { + mountDirHash := util.HashToInt32([]byte(dir)) + if mountDirHash < 0 { + mountDirHash = -mountDirHash + } + *option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash) + } + if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) { + glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error()) + } + montSocketListener, err := net.Listen("unix", *option.localSocket) + if err != nil { + glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err) + } // detect mount folder mode if *option.dirAutoCreate { @@ -129,6 +123,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { } fileInfo, err := os.Stat(dir) + // collect uid, gid uid, gid := uint32(0), uint32(0) mountMode := os.ModeDir | 0777 if err == nil { @@ -140,6 +135,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { return false } + // detect uid, gid if uid == 0 { if u, err := user.Current(); err == nil { if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil { @@ -165,34 +161,51 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { return true } - mountName := path.Base(dir) - - options := []fuse.MountOption{ - fuse.VolumeName(mountName), - fuse.FSName(*option.filer + ":" + filerMountRootPath), - fuse.Subtype("seaweedfs"), - // fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders - fuse.NoAppleXattr(), - fuse.ExclCreate(), - fuse.DaemonTimeout("3600"), - fuse.AllowSUID(), - fuse.DefaultPermissions(), - fuse.MaxReadahead(1024 * 128), - fuse.AsyncRead(), - fuse.WritebackCache(), - fuse.MaxBackground(128), - fuse.CongestionThreshold(128), - } - - options = append(options, osSpecificMountOptions()...) - if *option.allowOthers { - options = append(options, fuse.AllowOther()) + serverFriendlyName := strings.ReplaceAll(*option.filer, ",", "+") + + // mount fuse + fuseMountOptions := &fuse.MountOptions{ + AllowOther: *option.allowOthers, + Options: nil, + MaxBackground: 128, + MaxWrite: 1024 * 1024 * 2, + MaxReadAhead: 1024 * 1024 * 2, + IgnoreSecurityLabels: false, + RememberInodes: false, + FsName: serverFriendlyName + ":" + filerMountRootPath, + Name: "seaweedfs", + SingleThreaded: false, + DisableXAttrs: *option.disableXAttr, + Debug: *option.debug, + EnableLocks: false, + ExplicitDataCacheControl: false, + DirectMount: true, + DirectMountFlags: 0, + //SyncRead: false, // set to false to enable the FUSE_CAP_ASYNC_READ capability + //EnableAcl: true, } if *option.nonempty { - options = append(options, fuse.AllowNonEmptyMount()) + fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty") } if *option.readOnly { - options = append(options, fuse.ReadOnly()) + if runtime.GOOS == "darwin" { + fuseMountOptions.Options = append(fuseMountOptions.Options, "rdonly") + } else { + fuseMountOptions.Options = append(fuseMountOptions.Options, "ro") + } + } + if runtime.GOOS == "darwin" { + // https://github-wiki-see.page/m/macfuse/macfuse/wiki/Mount-Options + ioSizeMB := 1 + for ioSizeMB*2 <= *option.chunkSizeLimitMB && ioSizeMB*2 <= 32 { + ioSizeMB *= 2 + } + fuseMountOptions.Options = append(fuseMountOptions.Options, "daemon_timeout=600") + fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr") + // fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness + fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs") + fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName) + fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024)) } // find mount point @@ -201,60 +214,51 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { mountRoot = mountRoot[0 : len(mountRoot)-1] } - diskType := types.ToDiskType(*option.diskType) - - seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ + seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{ MountDirectory: dir, - FilerAddresses: filers, - FilerGrpcAddresses: filerGrpcAddresses, + FilerAddresses: filerAddresses, GrpcDialOption: grpcDialOption, FilerMountRootPath: mountRoot, Collection: *option.collection, Replication: *option.replication, TtlSec: int32(*option.ttlSec), - DiskType: diskType, + DiskType: types.ToDiskType(*option.diskType), ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, ConcurrentWriters: *option.concurrentWriters, CacheDir: *option.cacheDir, CacheSizeMB: *option.cacheSizeMB, DataCenter: *option.dataCenter, + Quota: int64(*option.collectionQuota) * 1024 * 1024, MountUid: uid, MountGid: gid, MountMode: mountMode, MountCtime: fileInfo.ModTime(), MountMtime: time.Now(), - MountParentInode: parentInode, Umask: umask, VolumeServerAccess: *mountOptions.volumeServerAccess, Cipher: cipher, UidGidMapper: uidGidMapper, + DisableXAttr: *option.disableXAttr, }) - // mount - c, err := fuse.Mount(dir, options...) + server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions) if err != nil { - glog.V(0).Infof("mount: %v", err) - return true + glog.Fatalf("Mount fail: %v", err) } - defer fuse.Unmount(dir) - grace.OnInterrupt(func() { - fuse.Unmount(dir) - c.Close() + unmount.Unmount(dir) }) - glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir) - server := fs.New(c, nil) - seaweedFileSystem.Server = server + grpcS := pb.NewGrpcServer() + mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem) + reflection.Register(grpcS) + go grpcS.Serve(montSocketListener) + seaweedFileSystem.StartBackgroundTasks() - err = server.Serve(seaweedFileSystem) - // check if the mount process has an error to report - <-c.Ready - if err := c.MountError; err != nil { - glog.V(0).Infof("mount process: %v", err) - return true - } + fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) + + server.Serve() return true } diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go index db0b4148d..3274f599b 100644 --- a/weed/command/msg_broker.go +++ b/weed/command/msg_broker.go @@ -3,7 +3,6 @@ package command import ( "context" "fmt" - "strconv" "time" "google.golang.org/grpc/reflection" @@ -63,35 +62,31 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool { grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile) - filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*msgBrokerOpt.filer) - if err != nil { - glog.Fatal(err) - return false - } + filerAddress := pb.ServerAddress(*msgBrokerOpt.filer) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker") cipher := false for { - err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) } cipher = resp.Cipher return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress) + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress()) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress) + glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress()) break } } qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{ - Filers: []string{*msgBrokerOpt.filer}, + Filers: []pb.ServerAddress{filerAddress}, DefaultReplication: "", MaxMB: 0, Ip: *msgBrokerOpt.ip, @@ -100,7 +95,7 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool { }, grpcDialOption) // start grpc listener - grpcL, err := util.NewListener(":"+strconv.Itoa(*msgBrokerOpt.port), 0) + grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0) if err != nil { glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err) } diff --git a/weed/command/s3.go b/weed/command/s3.go index c8292a7d5..42e447d90 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -3,11 +3,14 @@ package command import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "google.golang.org/grpc/reflection" "net/http" "time" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/gorilla/mux" @@ -23,26 +26,35 @@ var ( ) type S3Options struct { - filer *string - port *int - config *string - domainName *string - tlsPrivateKey *string - tlsCertificate *string - metricsHttpPort *int - allowEmptyFolder *bool + filer *string + bindIp *string + port *int + portGrpc *int + config *string + domainName *string + tlsPrivateKey *string + tlsCertificate *string + metricsHttpPort *int + allowEmptyFolder *bool + allowDeleteBucketNotEmpty *bool + auditLogConfig *string + localFilerSocket *string } func init() { cmdS3.Run = runS3 // break init cycle s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address") + s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.") s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port") + s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port") s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file") + s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file") s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file") s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file") s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") - s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", false, "allow empty folders") + s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders") + s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") } var cmdS3 = &Command{ @@ -137,11 +149,7 @@ func runS3(cmd *Command, args []string) bool { func (s3opt *S3Options) startS3Server() bool { - filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*s3opt.filer) - if err != nil { - glog.Fatal(err) - return false - } + filerAddress := pb.ServerAddress(*s3opt.filer) filerBucketsPath := "/buckets" @@ -152,10 +160,10 @@ func (s3opt *S3Options) startS3Server() bool { var metricsIntervalSec int for { - err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) } filerBucketsPath = resp.DirBuckets metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec) @@ -163,10 +171,10 @@ func (s3opt *S3Options) startS3Server() bool { return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress) + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress()) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress) + glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress()) break } } @@ -175,15 +183,16 @@ func (s3opt *S3Options) startS3Server() bool { router := mux.NewRouter().SkipClean(true) - _, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{ - Filer: *s3opt.filer, - Port: *s3opt.port, - FilerGrpcAddress: filerGrpcAddress, - Config: *s3opt.config, - DomainName: *s3opt.domainName, - BucketsPath: filerBucketsPath, - GrpcDialOption: grpcDialOption, - AllowEmptyFolder: *s3opt.allowEmptyFolder, + s3ApiServer, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{ + Filer: filerAddress, + Port: *s3opt.port, + Config: *s3opt.config, + DomainName: *s3opt.domainName, + BucketsPath: filerBucketsPath, + GrpcDialOption: grpcDialOption, + AllowEmptyFolder: *s3opt.allowEmptyFolder, + AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty, + LocalFilerSocket: s3opt.localFilerSocket, }) if s3ApiServer_err != nil { glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err) @@ -191,19 +200,61 @@ func (s3opt *S3Options) startS3Server() bool { httpS := &http.Server{Handler: router} - listenAddress := fmt.Sprintf(":%d", *s3opt.port) - s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second) + if *s3opt.portGrpc == 0 { + *s3opt.portGrpc = 10000 + *s3opt.port + } + if *s3opt.bindIp == "" { + *s3opt.bindIp = "localhost" + } + + listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port) + s3ApiListener, s3ApiLocalListner, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second) if err != nil { glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err) } + if len(*s3opt.auditLogConfig) > 0 { + s3err.InitAuditLog(*s3opt.auditLogConfig) + if s3err.Logger != nil { + defer s3err.Logger.Close() + } + } + + // starting grpc server + grpcPort := *s3opt.portGrpc + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0) + if err != nil { + glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err) + } + grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3")) + s3_pb.RegisterSeaweedS3Server(grpcS, s3ApiServer) + reflection.Register(grpcS) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } + go grpcS.Serve(grpcL) + if *s3opt.tlsPrivateKey != "" { glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port) + if s3ApiLocalListner != nil { + go func() { + if err = httpS.ServeTLS(s3ApiLocalListner, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil { + glog.Fatalf("S3 API Server Fail to serve: %v", err) + } + }() + } if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil { glog.Fatalf("S3 API Server Fail to serve: %v", err) } } else { glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port) + if s3ApiLocalListner != nil { + go func() { + if err = httpS.Serve(s3ApiLocalListner); err != nil { + glog.Fatalf("S3 API Server Fail to serve: %v", err) + } + }() + } if err = httpS.Serve(s3ApiListener); err != nil { glog.Fatalf("S3 API Server Fail to serve: %v", err) } diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 886c0ac5e..fb81f9966 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -2,9 +2,10 @@ package command import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/command/scaffold" - "io/ioutil" + "github.com/chrislusf/seaweedfs/weed/util" "path/filepath" + + "github.com/chrislusf/seaweedfs/weed/command/scaffold" ) func init() { @@ -55,7 +56,7 @@ func runScaffold(cmd *Command, args []string) bool { } if *outputPath != "" { - ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644) + util.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644) } else { fmt.Println(content) } diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index c8cb70131..c82de8da0 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -44,7 +44,7 @@ dbFile = "./filer.db" # sqlite db file # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # name VARCHAR(1000) BINARY COMMENT 'directory or file name', -# directory TEXT COMMENT 'full path to parent directory', +# directory TEXT BINARY COMMENT 'full path to parent directory', # meta LONGBLOB, # PRIMARY KEY (dirhash, name) # ) DEFAULT CHARSET=utf8; @@ -61,15 +61,15 @@ connection_max_lifetime_seconds = 0 interpolateParams = false # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: enableUpsert = true -upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" +upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" [mysql2] # or memsql, tidb enabled = false createTable = """ - CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( + CREATE TABLE IF NOT EXISTS `%s` ( dirhash BIGINT, name VARCHAR(1000) BINARY, - directory TEXT, + directory TEXT BINARY, meta LONGBLOB, PRIMARY KEY (dirhash, name) ) DEFAULT CHARSET=utf8; @@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0 interpolateParams = false # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: enableUpsert = true -upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" +upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" [postgres] # or cockroachdb, YugabyteDB # CREATE TABLE IF NOT EXISTS filemeta ( @@ -153,6 +153,8 @@ password = "" superLargeDirectories = [] # Name of the datacenter local to this filer, used as host selection fallback. localDC = "" +# Gocql connection timeout, default: 600ms +connection_timeout_millisecond = 600 [hbase] enabled = false @@ -167,6 +169,14 @@ database = 0 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss. superLargeDirectories = [] +[redis2_sentinel] +enabled = false +addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"] +masterName = "master" +username = "" +password = "" +database = 0 + [redis_cluster2] enabled = false addresses = [ @@ -185,6 +195,70 @@ routeByLatency = false # This changes the data layout. Only add new directories. Removing/Updating will cause data loss. superLargeDirectories = [] +[redis_lua] +enabled = false +address = "localhost:6379" +password = "" +database = 0 +# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. +superLargeDirectories = [] + +[redis_lua_sentinel] +enabled = false +addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"] +masterName = "master" +username = "" +password = "" +database = 0 + +[redis_lua_cluster] +enabled = false +addresses = [ + "localhost:30001", + "localhost:30002", + "localhost:30003", + "localhost:30004", + "localhost:30005", + "localhost:30006", +] +password = "" +# allows reads from slave servers or the master, but all writes still go to the master +readOnly = false +# automatically use the closest Redis server for reads +routeByLatency = false +# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. +superLargeDirectories = [] + +[redis3] # beta +enabled = false +address = "localhost:6379" +password = "" +database = 0 + +[redis3_sentinel] +enabled = false +addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"] +masterName = "master" +username = "" +password = "" +database = 0 + +[redis_cluster3] # beta +enabled = false +addresses = [ + "localhost:30001", + "localhost:30002", + "localhost:30003", + "localhost:30004", + "localhost:30005", + "localhost:30006", +] +password = "" +# allows reads from slave servers or the master, but all writes still go to the master +readOnly = false +# automatically use the closest Redis server for reads +routeByLatency = false + [etcd] enabled = false servers = "localhost:2379" @@ -211,6 +285,29 @@ healthcheck_enabled = false index.max_result_window = 10000 +[arangodb] # in development dont use it +enabled = false +db_name = "seaweedfs" +servers=["http://localhost:8529"] # list of servers to connect to +# only basic auth supported for now +username="" +password="" +# skip tls cert validation +insecure_skip_verify = true + +[ydb] # https://ydb.tech/ +enabled = false +dsn = "grpc://localhost:2136?database=/local" +prefix = "seaweedfs" +useBucketPrefix = true # Fast Bucket Deletion +poolSizeLimit = 50 +dialTimeOut = 10 + +# Authenticate produced with one of next environment variables: +# YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS= — used service account key file by path +# YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation +# YDB_METADATA_CREDENTIALS="1" — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function +# YDB_ACCESS_TOKEN_CREDENTIALS= — used for authenticate to YDB with short-life access token. For example, access token may be IAM token ########################## ########################## @@ -238,3 +335,5 @@ enabled = false pdaddrs = "localhost:2379" # Concurrency for TiKV delete range deleterange_concurrency = 1 +# Enable 1PC +enable_1pc = false diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index 020f48e36..10d9d1914 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -14,19 +14,14 @@ scripts = """ volume.deleteEmpty -quietFor=24h -force volume.balance -force volume.fix.replication + s3.clean.uploads -timeAgo=24h unlock """ sleep_minutes = 17 # sleep minutes between each script execution -[master.filer] -default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands - [master.sequencer] -type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence -# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence -# example : http://127.0.0.1:2379,http://127.0.0.1:2389 -sequencer_etcd_urls = "http://127.0.0.1:2379" +type = "raft" # Choose [raft|snowflake] type for storing the file id sequence # when sequencer.type = snowflake, the snowflake id must be different from other masters sequencer_snowflake_id = 0 # any number between 1~1023 @@ -41,6 +36,7 @@ aws_secret_access_key = "" # if empty, loads from the shared credentials fil region = "us-east-2" bucket = "your_bucket_name" # an existing bucket endpoint = "" +storage_class = "STANDARD_IA" # create this number of logical volumes if no more writable volumes # count_x means how many copies of data. diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index 0c69b2f24..e5452cdff 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -4,17 +4,46 @@ # /etc/seaweedfs/security.toml # this file is read by master, volume server, and filer -# the jwt signing key is read by master and volume server. -# a jwt defaults to expire after 10 seconds. +# this jwt signing key is read by master and volume server, and it is used for write operations: +# - the Master server generates the JWT, which can be used to write a certain file on a volume server +# - the Volume server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. [jwt.signing] key = "" expires_after_seconds = 10 # seconds -# jwt for read is only supported with master+volume setup. Filer does not support this mode. +# by default, if the signing key above is set, the Volume UI over HTTP is disabled. +# by setting ui.access to true, you can re-enable the Volume UI. Despite +# some information leakage (as the UI is not authenticated), this should not +# pose a security risk. +[access] +ui = false + +# this jwt signing key is read by master and volume server, and it is used for read operations: +# - the Master server generates the JWT, which can be used to read a certain file on a volume server +# - the Volume server validates the JWT on reading +# NOTE: jwt for read is only supported with master+volume setup. Filer does not support this mode. [jwt.signing.read] key = "" expires_after_seconds = 10 # seconds + +# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +[jwt.filer_signing] +key = "" +expires_after_seconds = 10 # seconds + +# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +[jwt.filer_signing.read] +key = "" +expires_after_seconds = 10 # seconds + # all grpc tls authentications are mutual # the values for the following ca, cert, and key are paths to the PERM files. # the host name is not checked, so the PERM files can be shared. @@ -38,6 +67,11 @@ cert = "" key = "" allowed_commonNames = "" # comma-separated SSL certificate common names +[grpc.s3] +cert = "" +key = "" +allowed_commonNames = "" # comma-separated SSL certificate common names + [grpc.msg_broker] cert = "" key = "" @@ -54,7 +88,13 @@ key = "" # this does not work with other clients, e.g., "weed filer|mount" etc, yet. [https.client] enabled = true + [https.volume] cert = "" key = "" +ca = "" +[https.master] +cert = "" +key = "" +ca = "" diff --git a/weed/command/server.go b/weed/command/server.go index c784d90b9..b1812bb9b 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -2,7 +2,6 @@ package command import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/util/grace" "net/http" "os" "strings" @@ -11,7 +10,9 @@ import ( stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/grace" ) type ServerOptions struct { @@ -27,6 +28,7 @@ var ( masterOptions MasterOptions filerOptions FilerOptions s3Options S3Options + iamOptions IamOptions webdavOptions WebDavOption msgBrokerOptions MessageBrokerOptions ) @@ -53,14 +55,14 @@ var cmdServer = &Command{ var ( serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier") - serverBindIp = cmdServer.Flag.String("ip.bind", "", "ip address to bind to") + serverBindIp = cmdServer.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.") serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.") volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") - volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.") + volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.") volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).") volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") @@ -70,6 +72,7 @@ var ( isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") + isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service") isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway") isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") @@ -85,6 +88,7 @@ func init() { serverOptions.debugPort = cmdServer.Flag.Int("debug.port", 6060, "http port for debugging") masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") + masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 0, "master server grpc listen port") masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list") masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") @@ -94,20 +98,26 @@ func init() { masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address") masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server") + masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") + masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers") + filerOptions.filerGroup = cmdServer.Flag.String("filer.filerGroup", "", "share metadata with other filers in the same filerGroup") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") + filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") - filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list") filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size") + filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-.sock") + filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") + serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.") serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") @@ -122,13 +132,19 @@ func init() { serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files") serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, " enable tcp port") + serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") + s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port") s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") - s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders") + s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") + s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders") + s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + + iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port") webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") @@ -137,7 +153,7 @@ func init() { webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") - webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") @@ -157,6 +173,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingS3 { *isStartingFiler = true } + if *isStartingIam { + *isStartingFiler = true + } if *isStartingWebDav { *isStartingFiler = true } @@ -165,20 +184,27 @@ func runServer(cmd *Command, args []string) bool { } if *isStartingMasterServer { - _, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers) - peers := strings.Join(peerList, ",") + _, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.portGrpc, *masterOptions.peers) + peers := strings.Join(pb.ToAddressStrings(peerList), ",") masterOptions.peers = &peers } + if *serverBindIp == "" { + serverBindIp = serverIp + } + // ip address masterOptions.ip = serverIp masterOptions.ipBind = serverBindIp - filerOptions.masters = masterOptions.peers + filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap() filerOptions.ip = serverIp filerOptions.bindIp = serverBindIp + s3Options.bindIp = serverBindIp + iamOptions.ip = serverBindIp + iamOptions.masters = masterOptions.peers serverOptions.v.ip = serverIp serverOptions.v.bindIp = serverBindIp - serverOptions.v.masters = masterOptions.peers + serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses() serverOptions.v.idleConnectionTimeout = serverTimeout serverOptions.v.dataCenter = serverDataCenter serverOptions.v.rack = serverRack @@ -194,8 +220,9 @@ func runServer(cmd *Command, args []string) bool { filerOptions.disableHttp = serverDisableHttp masterOptions.disableHttp = serverDisableHttp - filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port) + filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc)) s3Options.filer = &filerAddress + iamOptions.filer = &filerAddress webdavOptions.filer = &filerAddress msgBrokerOptions.filer = &filerAddress @@ -222,25 +249,28 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { time.Sleep(1 * time.Second) - filerOptions.startFiler() - }() } if *isStartingS3 { go func() { time.Sleep(2 * time.Second) - + s3Options.localFilerSocket = filerOptions.localSocket s3Options.startS3Server() + }() + } + if *isStartingIam { + go func() { + time.Sleep(2 * time.Second) + iamOptions.startIamServer() }() } if *isStartingWebDav { go func() { time.Sleep(2 * time.Second) - webdavOptions.startWebDav() }() diff --git a/weed/command/shell.go b/weed/command/shell.go index 4a9f4b027..c32a8e614 100644 --- a/weed/command/shell.go +++ b/weed/command/shell.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/shell" @@ -17,6 +18,7 @@ var ( func init() { cmdShell.Run = runShell // break init cycle shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333") + shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers") shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888") shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml") } @@ -36,7 +38,7 @@ func runShell(command *Command, args []string) bool { util.LoadConfiguration("security", false) shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") - if *shellOptions.Masters == "" && *shellInitialFiler == "" { + if *shellOptions.Masters == "" { util.LoadConfiguration("shell", false) v := util.GetViper() cluster := v.GetString("cluster.default") @@ -44,22 +46,15 @@ func runShell(command *Command, args []string) bool { cluster = *shellCluster } if cluster == "" { - *shellOptions.Masters, *shellInitialFiler = "localhost:9333", "localhost:8888" + *shellOptions.Masters = "localhost:9333" } else { *shellOptions.Masters = v.GetString("cluster." + cluster + ".master") *shellInitialFiler = v.GetString("cluster." + cluster + ".filer") + fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler) } } - fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler) - - var err error - shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(*shellInitialFiler) - shellOptions.FilerAddress = *shellInitialFiler - if err != nil { - fmt.Printf("failed to parse filer %s: %v\n", *shellInitialFiler, err) - return false - } + shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler) shellOptions.Directory = "/" shell.RunShell(shellOptions) diff --git a/weed/command/update.go b/weed/command/update.go new file mode 100644 index 000000000..2d0dc42ad --- /dev/null +++ b/weed/command/update.go @@ -0,0 +1,382 @@ +package command + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "golang.org/x/net/context/ctxhttp" +) + +//copied from https://github.com/restic/restic/tree/master/internal/selfupdate + +// Release collects data about a single release on GitHub. +type Release struct { + Name string `json:"name"` + TagName string `json:"tag_name"` + Draft bool `json:"draft"` + PreRelease bool `json:"prerelease"` + PublishedAt time.Time `json:"published_at"` + Assets []Asset `json:"assets"` + + Version string `json:"-"` // set manually in the code +} + +// Asset is a file uploaded and attached to a release. +type Asset struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` +} + +const githubAPITimeout = 30 * time.Second + +// githubError is returned by the GitHub API, e.g. for rate-limiting. +type githubError struct { + Message string +} + +//default version is not full version +var isFullVersion = false + +var ( + updateOpt UpdateOptions +) + +type UpdateOptions struct { + dir *string + name *string + Version *string +} + +func init() { + path, _ := os.Executable() + _, name := filepath.Split(path) + updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.") + updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed. On windows, name shouldn't be same to the orignial name.") + updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.") + cmdUpdate.Run = runUpdate +} + +var cmdUpdate = &Command{ + UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]", + Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs", + Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`, +} + +func runUpdate(cmd *Command, args []string) bool { + path, _ := os.Executable() + _, name := filepath.Split(path) + + if *updateOpt.dir != "" { + if err := util.TestFolderWritable(util.ResolvePath(*updateOpt.dir)); err != nil { + glog.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err) + return false + } + } else { + *updateOpt.dir = filepath.Dir(path) + } + + if *updateOpt.name == "" { + *updateOpt.name = name + } + + target := filepath.Join(*updateOpt.dir, *updateOpt.name) + + if runtime.GOOS == "windows" { + if target == path { + glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.") + return false + } + } + + glog.V(0).Infof("new weed will be saved to %s", target) + + _, err := downloadRelease(context.Background(), target, *updateOpt.Version) + if err != nil { + glog.Errorf("unable to download weed: %v", err) + return false + } + return true +} + +func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) { + currentVersion := util.VERSION_NUMBER + rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs") + if err != nil { + return "", err + } + + if rel.Version == currentVersion { + if ver == "0" { + glog.V(0).Infof("weed is up to date") + } else { + glog.V(0).Infof("no need to download the same version of weed ") + } + return currentVersion, nil + } + + glog.V(0).Infof("download version: %s", rel.Version) + + largeDiskSuffix := "" + if util.VolumeSizeLimitGB == 8000 { + largeDiskSuffix = "_large_disk" + } + + fullSuffix := "" + if isFullVersion { + fullSuffix = "_full" + } + + ext := "tar.gz" + if runtime.GOOS == "windows" { + ext = "zip" + } + + suffix := fmt.Sprintf("%s_%s%s%s.%s", runtime.GOOS, runtime.GOARCH, fullSuffix, largeDiskSuffix, ext) + md5Filename := fmt.Sprintf("%s.md5", suffix) + _, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename) + if err != nil { + return "", err + } + + downloadFilename, buf, err := getGithubDataFile(ctx, rel.Assets, suffix) + if err != nil { + return "", err + } + + md5Ctx := md5.New() + md5Ctx.Write(buf) + binaryMd5 := md5Ctx.Sum(nil) + if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) { + glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32])) + err = fmt.Errorf("binary md5sum doesn't match") + return "", err + } + + err = extractToFile(buf, downloadFilename, target) + if err != nil { + return "", err + } else { + glog.V(0).Infof("successfully updated weed to version %v\n", rel.Version) + } + + return rel.Version, nil +} + +// GitHubLatestRelease uses the GitHub API to get information about the specific +// release of a repository. +func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (Release, error) { + ctx, cancel := context.WithTimeout(ctx, githubAPITimeout) + defer cancel() + + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return Release{}, err + } + + // pin API version 3 + req.Header.Set("Accept", "application/vnd.github.v3+json") + + res, err := ctxhttp.Do(ctx, http.DefaultClient, req) + if err != nil { + return Release{}, err + } + + if res.StatusCode != http.StatusOK { + content := res.Header.Get("Content-Type") + if strings.Contains(content, "application/json") { + // try to decode error message + var msg githubError + jerr := json.NewDecoder(res.Body).Decode(&msg) + if jerr == nil { + return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message) + } + } + + _ = res.Body.Close() + return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + _ = res.Body.Close() + return Release{}, err + } + + err = res.Body.Close() + if err != nil { + return Release{}, err + } + + var release Release + var releaseList []Release + err = json.Unmarshal(buf, &releaseList) + if err != nil { + return Release{}, err + } + if ver == "0" { + release = releaseList[0] + glog.V(0).Infof("latest version is %v\n", release.TagName) + } else { + for _, r := range releaseList { + if r.TagName == ver { + release = r + break + } + } + } + + if release.TagName == "" { + return Release{}, fmt.Errorf("can not find the specific version") + } + + release.Version = release.TagName + return release, nil +} + +func getGithubData(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + // request binary data + req.Header.Set("Accept", "application/octet-stream") + + res, err := ctxhttp.Do(ctx, http.DefaultClient, req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + _ = res.Body.Close() + return nil, err + } + + err = res.Body.Close() + if err != nil { + return nil, err + } + + return buf, nil +} + +func getGithubDataFile(ctx context.Context, assets []Asset, suffix string) (filename string, data []byte, err error) { + var url string + for _, a := range assets { + if strings.HasSuffix(a.Name, suffix) { + url = a.URL + filename = a.Name + break + } + } + + if url == "" { + return "", nil, fmt.Errorf("unable to find file with suffix %v", suffix) + } + + glog.V(0).Infof("download %v\n", filename) + data, err = getGithubData(ctx, url) + if err != nil { + return "", nil, err + } + + return filename, data, nil +} + +func extractToFile(buf []byte, filename, target string) error { + var rd io.Reader = bytes.NewReader(buf) + + switch filepath.Ext(filename) { + case ".gz": + gr, err := gzip.NewReader(rd) + if err != nil { + return err + } + defer gr.Close() + trd := tar.NewReader(gr) + hdr, terr := trd.Next() + if terr != nil { + glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr) + return terr + } + rd = trd + case ".zip": + zrd, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) + if err != nil { + return err + } + + if len(zrd.File) != 1 { + return fmt.Errorf("ZIP archive contains more than one file") + } + + file, err := zrd.File[0].Open() + if err != nil { + return err + } + + defer func() { + _ = file.Close() + }() + + rd = file + } + + // Write everything to a temp file + dir := filepath.Dir(target) + new, err := ioutil.TempFile(dir, "weed") + if err != nil { + return err + } + + n, err := io.Copy(new, rd) + if err != nil { + _ = new.Close() + _ = os.Remove(new.Name()) + return err + } + if err = new.Sync(); err != nil { + return err + } + if err = new.Close(); err != nil { + return err + } + + mode := os.FileMode(0755) + // attempt to find the original mode + if fi, err := os.Lstat(target); err == nil { + mode = fi.Mode() + } + + // Rename the temp file to the final location atomically. + if err := os.Rename(new.Name(), target); err != nil { + return err + } + + glog.V(0).Infof("saved %d bytes in %v\n", n, target) + return os.Chmod(target, mode) +} diff --git a/weed/command/update_full.go b/weed/command/update_full.go new file mode 100644 index 000000000..529f38219 --- /dev/null +++ b/weed/command/update_full.go @@ -0,0 +1,9 @@ +//go:build elastic && ydb && gocdk && hdfs +// +build elastic,ydb,gocdk,hdfs + +package command + +//set true if gtags are set +func init() { + isFullVersion = true +} diff --git a/weed/command/upload.go b/weed/command/upload.go index 9ae1befab..f2b0b7fe4 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -71,7 +71,7 @@ func runUpload(cmd *Command, args []string) bool { util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") - defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master) + defaultReplication, err := readMasterConfiguration(grpcDialOption, pb.ServerAddress(*upload.master)) if err != nil { fmt.Printf("upload: %v", err) return false @@ -96,7 +96,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { return e } - results, e := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl) + results, e := operation.SubmitFiles(func() pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl) bytes, _ := json.Marshal(results) fmt.Println(string(bytes)) if e != nil { @@ -118,7 +118,7 @@ func runUpload(cmd *Command, args []string) bool { fmt.Println(e.Error()) return false } - results, err := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl) + results, err := operation.SubmitFiles(func() pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl) if err != nil { fmt.Println(err.Error()) return false @@ -129,8 +129,8 @@ func runUpload(cmd *Command, args []string) bool { return true } -func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress string) (replication string, err error) { - err = pb.WithMasterClient(masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error { +func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress pb.ServerAddress) (replication string, err error) { + err = pb.WithMasterClient(false, masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error { resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) if err != nil { return fmt.Errorf("get master %s configuration: %v", masterAddress, err) diff --git a/weed/command/volume.go b/weed/command/volume.go index 235eff11b..158bdf162 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -2,7 +2,6 @@ package command import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" "net/http" httppprof "net/http/pprof" "os" @@ -11,6 +10,8 @@ import ( "strings" "time" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/spf13/viper" "google.golang.org/grpc" @@ -24,7 +25,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" - "github.com/chrislusf/seaweedfs/weed/server" + weed_server "github.com/chrislusf/seaweedfs/weed/server" stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" @@ -36,6 +37,7 @@ var ( type VolumeServerOptions struct { port *int + portGrpc *int publicPort *int folders []string folderMaxLimits []int @@ -43,7 +45,8 @@ type VolumeServerOptions struct { ip *string publicUrl *string bindIp *string - masters *string + mastersString *string + masters []pb.ServerAddress idleConnectionTimeout *int dataCenter *string rack *string @@ -62,17 +65,19 @@ type VolumeServerOptions struct { preStopSeconds *int metricsHttpPort *int // pulseSeconds *int - enableTcp *bool + enableTcp *bool + inflightUploadDataTimeout *time.Duration } func init() { cmdVolume.Run = runVolume // break init cycle v.port = cmdVolume.Flag.Int("port", 8080, "http listen port") + v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port") v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public") v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier") v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") - v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to") - v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers") + v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.") + v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers") v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds") @@ -91,7 +96,8 @@ func init() { v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files") - v.enableTcp = cmdVolume.Flag.Bool("tcp", false, " enable tcp port") + v.enableTcp = cmdVolume.Flag.Bool("tcp", false, " enable tcp port") + v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") } var cmdVolume = &Command{ @@ -104,7 +110,7 @@ var cmdVolume = &Command{ var ( volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") - maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.") + maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.") volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).") minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") @@ -123,6 +129,7 @@ func runVolume(cmd *Command, args []string) bool { go stats_collect.StartMetricsServer(*v.metricsHttpPort) minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent) + v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses() v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces) return true @@ -189,12 +196,18 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v *v.ip = util.DetectedHostAddress() glog.V(0).Infof("detected volume server ip address: %v", *v.ip) } + if *v.bindIp == "" { + *v.bindIp = *v.ip + } if *v.publicPort == 0 { *v.publicPort = *v.port } + if *v.portGrpc == 0 { + *v.portGrpc = 10000 + *v.port + } if *v.publicUrl == "" { - *v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort) + *v.publicUrl = util.JoinHostPort(*v.ip, *v.publicPort) } volumeMux := http.NewServeMux() @@ -221,20 +234,19 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v volumeNeedleMapKind = storage.NeedleMapLevelDbLarge } - masters := *v.masters - volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, - *v.ip, *v.port, *v.publicUrl, + *v.ip, *v.port, *v.portGrpc, *v.publicUrl, v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, *v.idxFolder, volumeNeedleMapKind, - strings.Split(masters, ","), 5, *v.dataCenter, *v.rack, + v.masters, 5, *v.dataCenter, *v.rack, v.whiteList, *v.fixJpgOrientation, *v.readMode, *v.compactionMBPerSecond, *v.fileSizeLimitMB, int64(*v.concurrentUploadLimitMB)*1024*1024, int64(*v.concurrentDownloadLimitMB)*1024*1024, + *v.inflightUploadDataTimeout, ) // starting grpc server grpcS := v.startGrpcService(volumeServer) @@ -258,7 +270,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v stopChan := make(chan bool) grace.OnInterrupt(func() { - fmt.Println("volume server has be killed") + fmt.Println("volume server has been killed") // Stop heartbeats if !volumeServer.StopHeartbeat() { @@ -307,8 +319,8 @@ func (v VolumeServerOptions) isSeparatedPublicPort() bool { } func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server { - grpcPort := *v.port + 10000 - grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0) + grpcPort := *v.portGrpc + grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0) if err != nil { glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err) } @@ -324,7 +336,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe } func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server { - publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort) + publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort) glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress) publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { @@ -351,7 +363,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd keyFile = viper.GetString("https.volume.key") } - listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port) + listeningAddress := util.JoinHostPort(*v.bindIp, *v.port) glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress) listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { @@ -359,11 +371,18 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd } httpDown := httpdown.HTTP{ - KillTimeout: 5 * time.Minute, - StopTimeout: 5 * time.Minute, + KillTimeout: time.Minute, + StopTimeout: 30 * time.Second, CertFile: certFile, KeyFile: keyFile} - clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener) + httpS := &http.Server{Handler: handler} + + if viper.GetString("https.volume.ca") != "" { + clientCertFile := viper.GetString("https.volume.ca") + httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile) + } + + clusterHttpServer := httpDown.Serve(httpS, listener) go func() { if e := clusterHttpServer.Wait(); e != nil { glog.Fatalf("Volume server fail to serve: %v", e) @@ -373,7 +392,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd } func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) { - listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port+20000) + listeningAddress := util.JoinHostPort(*v.bindIp, *v.port+20000) glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress) listener, e := util.NewListener(listeningAddress, 0) if e != nil { diff --git a/weed/command/webdav.go b/weed/command/webdav.go index 781ea1e36..689bf3c30 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -43,7 +43,7 @@ func init() { webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file") webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") - webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB") + webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB") } var cmdWebDav = &Command{ @@ -78,46 +78,41 @@ func (wo *WebDavOption) startWebDav() bool { } // parse filer grpc address - filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*wo.filer) - if err != nil { - glog.Fatal(err) - return false - } + filerAddress := pb.ServerAddress(*wo.filer) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var cipher bool // connect to filer for { - err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) } cipher = resp.Cipher return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress()) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress()) break } } ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{ - Filer: *wo.filer, - FilerGrpcAddress: filerGrpcAddress, - GrpcDialOption: grpcDialOption, - Collection: *wo.collection, - Replication: *wo.replication, - DiskType: *wo.disk, - Uid: uid, - Gid: gid, - Cipher: cipher, - CacheDir: util.ResolvePath(*wo.cacheDir), - CacheSizeMB: *wo.cacheSizeMB, + Filer: filerAddress, + GrpcDialOption: grpcDialOption, + Collection: *wo.collection, + Replication: *wo.replication, + DiskType: *wo.disk, + Uid: uid, + Gid: gid, + Cipher: cipher, + CacheDir: util.ResolvePath(*wo.cacheDir), + CacheSizeMB: *wo.cacheSizeMB, }) if webdavServer_err != nil { glog.Fatalf("WebDav Server startup error: %v", webdavServer_err) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 4bf9b16fa..13268b944 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -156,7 +156,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index 03b016c76..aaf1c196c 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -18,7 +18,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by return fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value) if err == nil { @@ -53,7 +53,7 @@ func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []b return nil, fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) row := db.QueryRowContext(ctx, store.GetSqlFind(DEFAULT_TABLE), dirHash, name, dirStr) err = row.Scan(&value) @@ -76,7 +76,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er return fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) res, err := db.ExecContext(ctx, store.GetSqlDelete(DEFAULT_TABLE), dirHash, name, dirStr) if err != nil { @@ -92,7 +92,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er } -func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) { +func GenDirAndName(key []byte) (dirStr string, dirHash int64, name string) { for len(key) < 8 { key = append(key, 0) } diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go new file mode 100644 index 000000000..13d14b2b0 --- /dev/null +++ b/weed/filer/arangodb/arangodb_store.go @@ -0,0 +1,347 @@ +package arangodb + +import ( + "context" + "crypto/tls" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + filer.Stores = append(filer.Stores, &ArangodbStore{}) +} + +var ( + BUCKET_PREFIX = "/buckets" + DEFAULT_COLLECTION = "seaweed_no_bucket" + KVMETA_COLLECTION = "seaweed_kvmeta" +) + +type ArangodbStore struct { + connect driver.Connection + client driver.Client + database driver.Database + kvCollection driver.Collection + + buckets map[string]driver.Collection + mu sync.RWMutex + + databaseName string +} + +type Model struct { + Key string `json:"_key"` + Directory string `json:"directory,omitempty"` + Name string `json:"name,omitempty"` + Ttl string `json:"ttl,omitempty"` + + //arangodb does not support binary blobs + //we encode byte slice into uint64 slice + //see helpers.go + Meta []uint64 `json:"meta"` +} + +func (store *ArangodbStore) GetName() string { + return "arangodb" +} + +func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + store.buckets = make(map[string]driver.Collection, 3) + store.databaseName = configuration.GetString(prefix + "db_name") + return store.connection(configuration.GetStringSlice(prefix+"servers"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetBool(prefix+"insecure_skip_verify"), + ) +} + +func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + + store.connect, err = http.NewConnection(http.ConnectionConfig{ + Endpoints: uris, + TLSConfig: &tls.Config{ + InsecureSkipVerify: insecure, + }, + }) + if err != nil { + return err + } + store.client, err = driver.NewClient(driver.ClientConfig{ + Connection: store.connect, + Authentication: driver.BasicAuthentication(user, pass), + }) + if err != nil { + return err + } + ok, err := store.client.DatabaseExists(ctx, store.databaseName) + if err != nil { + return err + } + if ok { + store.database, err = store.client.Database(ctx, store.databaseName) + } else { + store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{}) + } + if err != nil { + return err + } + if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil { + return err + } + return err +} + +type key int + +const ( + transactionKey key = 0 +) + +func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + keys := make([]string, 0, len(store.buckets)+1) + for k := range store.buckets { + keys = append(keys, k) + } + keys = append(keys, store.kvCollection.Name()) + txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{ + Exclusive: keys, + }, &driver.BeginTransactionOptions{}) + if err != nil { + return nil, err + } + + return context.WithValue(ctx, transactionKey, txn), nil +} + +func (store *ArangodbStore) CommitTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339) + } else { + model.Ttl = "" + } + + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.CreateDocument(ctx, model) + if driver.IsConflict(err) { + return store.UpdateEntry(ctx, entry) + } + + if err != nil { + return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err) + } + + return nil + +} + +func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339) + } else { + model.Ttl = "none" + } + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.UpdateDocument(ctx, model.Key, model) + if err != nil { + return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + } + + return nil +} + +func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + var data Model + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return nil, err + } + _, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data) + if err != nil { + if driver.IsNotFound(err) { + return nil, filer_pb.ErrNotFound + } + glog.Errorf("find %s: %v", fullpath, err) + return nil, filer_pb.ErrNotFound + } + if len(data.Meta) == 0 { + return nil, filer_pb.ErrNotFound + } + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } + _, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath))) + if err != nil && !driver.IsNotFound(err) { + glog.Errorf("find %s: %v", fullpath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) + } + return nil +} + +// this runs in log time +func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + var query string + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } + query = query + fmt.Sprintf(` + for d in %s + filter starts_with(d.directory, "%s/") || d.directory == "%s" + remove d._key in %s`, + targetCollection.Name(), + strings.Join(strings.Split(string(fullpath), "/"), ","), + string(fullpath), + targetCollection.Name(), + ) + cur, err := store.database.Query(ctx, query, nil) + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + defer cur.Close() + return nil +} + +func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) +} + +func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/") + if err != nil { + return lastFileName, err + } + query := "for d in " + targetCollection.Name() + if includeStartFile { + query = query + " filter d.name >= \"" + startFileName + "\" " + } else { + query = query + " filter d.name > \"" + startFileName + "\" " + } + if prefix != "" { + query = query + fmt.Sprintf(`&& starts_with(d.name, "%s")`, prefix) + } + query = query + ` +filter d.directory == @dir +sort d.name asc +` + if limit > 0 { + query = query + "limit " + strconv.Itoa(int(limit)) + } + query = query + "\n return d" + cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath}) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + defer cur.Close() + for cur.HasMore() { + var data Model + _, err = cur.ReadDocument(ctx, &data) + if err != nil { + break + } + entry := &filer.Entry{ + FullPath: util.NewFullPath(data.Directory, data.Name), + } + lastFileName = data.Name + converted := arrayToBytes(data.Meta) + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + + if !eachEntryFunc(entry) { + break + } + + } + return lastFileName, err +} + +func (store *ArangodbStore) Shutdown() { +} diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go new file mode 100644 index 000000000..810d639a7 --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -0,0 +1,40 @@ +package arangodb + +import ( + "context" + "github.com/arangodb/go-driver" + "time" + + "github.com/chrislusf/seaweedfs/weed/filer" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +var _ filer.BucketAware = (*ArangodbStore)(nil) + +func (store *ArangodbStore) OnBucketCreation(bucket string) { + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // create the collection && add to cache + _, err := store.ensureBucket(timeout, bucket) + if err != nil { + glog.Errorf("bucket create %s: %v", bucket, err) + } +} +func (store *ArangodbStore) OnBucketDeletion(bucket string) { + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + collection, err := store.ensureBucket(timeout, bucket) + if err != nil { + glog.Errorf("bucket delete %s: %v", bucket, err) + return + } + err = collection.Remove(timeout) + if err != nil && !driver.IsNotFound(err) { + glog.Errorf("bucket delete %s: %v", bucket, err) + return + } +} +func (store *ArangodbStore) CanDropWholeBucket() bool { + return true +} diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go new file mode 100644 index 000000000..c1307e78d --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -0,0 +1,54 @@ +package arangodb + +import ( + "context" + "fmt" + + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + model := &Model{ + Key: hashString(".kvstore." + string(key)), + Directory: ".kvstore." + string(key), + Meta: bytesToArray(value), + } + + exists, err := store.kvCollection.DocumentExists(ctx, model.Key) + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + if exists { + _, err = store.kvCollection.UpdateDocument(ctx, model.Key, model) + } else { + _, err = store.kvCollection.CreateDocument(ctx, model) + } + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} +func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + var model Model + _, err = store.kvCollection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) + if driver.IsNotFound(err) { + return nil, filer.ErrKvNotFound + } + if err != nil { + glog.Errorf("kv get: %s %v", string(key), err) + return nil, filer.ErrKvNotFound + } + return arrayToBytes(model.Meta), nil +} + +func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { + _, err = store.kvCollection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) + if err != nil { + glog.Errorf("kv del: %v", err) + return filer.ErrKvNotFound + } + return nil +} diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go new file mode 100644 index 000000000..943189781 --- /dev/null +++ b/weed/filer/arangodb/helpers.go @@ -0,0 +1,136 @@ +package arangodb + +import ( + "context" + "crypto/md5" + "encoding/binary" + "encoding/hex" + "io" + "strings" + + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/util" +) + +//convert a string into arango-key safe hex bytes hash +func hashString(dir string) string { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return hex.EncodeToString(b) +} + +// convert slice of bytes into slice of uint64 +// the first uint64 indicates the length in bytes +func bytesToArray(bs []byte) []uint64 { + out := make([]uint64, 0, 2+len(bs)/8) + out = append(out, uint64(len(bs))) + for len(bs)%8 != 0 { + bs = append(bs, 0) + } + for i := 0; i < len(bs); i = i + 8 { + out = append(out, binary.BigEndian.Uint64(bs[i:])) + } + return out +} + +// convert from slice of uint64 back to bytes +// if input length is 0 or 1, will return nil +func arrayToBytes(xs []uint64) []byte { + if len(xs) < 2 { + return nil + } + first := xs[0] + out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts... + for i := 1; i < len(xs); i = i + 1 { + binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) + } + return out[:first] +} + +// gets the collection the bucket points to from filepath +func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) { + bucket, _ := extractBucket(fullpath) + if bucket == "" { + bucket = DEFAULT_COLLECTION + } + c, err = store.ensureBucket(ctx, bucket) + if err != nil { + return nil, err + } + return c, err +} + +// called by extractBucketCollection +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { + return "", string(fullpath) + } + if strings.Count(string(fullpath), "/") < 3 { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + +// get bucket collection from cache. if not exist, creates the buckets collection and grab it +func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) { + var ok bool + store.mu.RLock() + bc, ok = store.buckets[bucket] + store.mu.RUnlock() + if ok { + return bc, nil + } + store.mu.Lock() + defer store.mu.Unlock() + store.buckets[bucket], err = store.ensureCollection(ctx, bucket) + if err != nil { + return nil, err + } + return store.buckets[bucket], nil +} + +// creates collection if not exist, ensures indices if not exist +func (store *ArangodbStore) ensureCollection(ctx context.Context, name string) (c driver.Collection, err error) { + ok, err := store.database.CollectionExists(ctx, name) + if err != nil { + return + } + if ok { + c, err = store.database.Collection(ctx, name) + } else { + c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{}) + } + if err != nil { + return + } + // ensure indices + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"}, + &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", Unique: true, + }); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"}, + &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { + return + } + if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1, + &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_name", + }); err != nil { + return + } + return c, nil +} diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md new file mode 100644 index 000000000..e189811fb --- /dev/null +++ b/weed/filer/arangodb/readme.md @@ -0,0 +1,52 @@ +##arangodb + +database: https://github.com/arangodb/arangodb +go driver: https://github.com/arangodb/go-driver + +options: + +``` +[arangodb] +enabled=true +db_name="seaweedfs" +servers=["http://localhost:8529"] +#basic auth +user="root" +pass="test" + +# tls settings +insecure_skip_verify=true +``` + +i test using this dev database: +`docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` + + +## features i don't personally need but are missing + [ ] provide tls cert to arango + [ ] authentication that is not basic auth + [ ] synchronise endpoint interval config + [ ] automatic creation of custom index + [ ] configure default arangodb collection sharding rules + [ ] configure default arangodb collection replication rules + + +## complexity + +ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct + +O(1) +- InsertEntry +- UpdateEntry +- FindEntry +- DeleteEntry +- KvPut +- KvGet +- KvDelete + +O(log(BUCKET_SIZE)) +- DeleteFolderChildren + +O(log(DIRECTORY_SIZE)) +- ListDirectoryEntries +- ListDirectoryPrefixedEntries diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index fc0b52ac7..d8c094a45 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/gocql/gocql" + "time" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -33,6 +34,7 @@ func (store *CassandraStore) Initialize(configuration util.Configuration, prefix configuration.GetString(prefix+"password"), configuration.GetStringSlice(prefix+"superLargeDirectories"), configuration.GetString(prefix+"localDC"), + configuration.GetInt(prefix+"connection_timeout_millisecond"), ) } @@ -41,12 +43,14 @@ func (store *CassandraStore) isSuperLargeDirectory(dir string) (dirHash string, return } -func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string) (err error) { +func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string, timeout int) (err error) { store.cluster = gocql.NewCluster(hosts...) if username != "" && password != "" { store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password} } store.cluster.Keyspace = keyspace + store.cluster.Timeout = time.Duration(timeout) * time.Millisecond + glog.V(0).Infof("timeout = %d", timeout) fallback := gocql.RoundRobinHostPolicy() if localDC != "" { fallback = gocql.DCAwareRoundRobinPolicy(localDC) @@ -96,7 +100,7 @@ func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } diff --git a/weed/filer/configuration.go b/weed/filer/configuration.go index 9ef2f3e0f..85fc65d13 100644 --- a/weed/filer/configuration.go +++ b/weed/filer/configuration.go @@ -12,7 +12,7 @@ var ( Stores []FilerStore ) -func (f *Filer) LoadConfiguration(config *util.ViperProxy) { +func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) { validateOneEnabledStore(config) @@ -24,7 +24,7 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) { if err := store.Initialize(config, store.GetName()+"."); err != nil { glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err) } - f.SetStore(store) + isFresh = f.SetStore(store) glog.V(0).Infof("configured filer store to %s", store.GetName()) hasDefaultStoreConfigured = true break @@ -77,6 +77,7 @@ func (f *Filer) LoadConfiguration(config *util.ViperProxy) { glog.V(0).Infof("configure filer %s for %s", store.GetName(), location) } + return } func validateOneEnabledStore(config *util.ViperProxy) { diff --git a/weed/filer/elastic/v7/doc.go b/weed/filer/elastic/v7/doc.go new file mode 100644 index 000000000..704bbf6de --- /dev/null +++ b/weed/filer/elastic/v7/doc.go @@ -0,0 +1,9 @@ +/* + +Package elastic is for elastic filer store. + +The referenced "github.com/olivere/elastic/v7" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package elastic diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index a16e5ebca..cb2c66f5a 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/filer/elastic/v7/elastic_store_kv.go b/weed/filer/elastic/v7/elastic_store_kv.go index 99c03314e..43835c153 100644 --- a/weed/filer/elastic/v7/elastic_store_kv.go +++ b/weed/filer/elastic/v7/elastic_store_kv.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/filer/entry.go b/weed/filer/entry.go index 8fa75fe6b..8dd00f010 100644 --- a/weed/filer/entry.go +++ b/weed/filer/entry.go @@ -15,15 +15,14 @@ type Attr struct { Uid uint32 // owner uid Gid uint32 // group gid Mime string // mime type - Replication string // replication - Collection string // collection name TtlSec int32 // ttl in seconds - DiskType string UserName string GroupNames []string SymlinkTarget string Md5 []byte FileSize uint64 + Rdev uint32 + Inode uint64 } func (attr Attr) IsDirectory() bool { @@ -43,6 +42,7 @@ type Entry struct { HardLinkCounter int32 Content []byte Remote *filer_pb.RemoteEntry + Quota int64 } func (entry *Entry) Size() uint64 { @@ -70,6 +70,7 @@ func (entry *Entry) ShallowClone() *Entry { newEntry.HardLinkCounter = entry.HardLinkCounter newEntry.Content = entry.Content newEntry.Remote = entry.Remote + newEntry.Quota = entry.Quota return newEntry } @@ -96,6 +97,7 @@ func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) { message.HardLinkCounter = entry.HardLinkCounter message.Content = entry.Content message.RemoteEntry = entry.Remote + message.Quota = entry.Quota } func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) { @@ -106,6 +108,7 @@ func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) { fsEntry.HardLinkCounter = message.HardLinkCounter fsEntry.Content = message.Content fsEntry.Remote = message.RemoteEntry + fsEntry.Quota = message.Quota } func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry { diff --git a/weed/filer/entry_codec.go b/weed/filer/entry_codec.go index 55c937b39..3d29ba0b4 100644 --- a/weed/filer/entry_codec.go +++ b/weed/filer/entry_codec.go @@ -39,15 +39,14 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes { Uid: entry.Uid, Gid: entry.Gid, Mime: entry.Mime, - Collection: entry.Attr.Collection, - Replication: entry.Attr.Replication, TtlSec: entry.Attr.TtlSec, - DiskType: entry.Attr.DiskType, UserName: entry.Attr.UserName, GroupName: entry.Attr.GroupNames, SymlinkTarget: entry.Attr.SymlinkTarget, Md5: entry.Attr.Md5, FileSize: entry.Attr.FileSize, + Rdev: entry.Attr.Rdev, + Inode: entry.Attr.Inode, } } @@ -65,15 +64,14 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr { t.Uid = attr.Uid t.Gid = attr.Gid t.Mime = attr.Mime - t.Collection = attr.Collection - t.Replication = attr.Replication t.TtlSec = attr.TtlSec - t.DiskType = attr.DiskType t.UserName = attr.UserName t.GroupNames = attr.GroupName t.SymlinkTarget = attr.SymlinkTarget t.Md5 = attr.Md5 t.FileSize = attr.FileSize + t.Rdev = attr.Rdev + t.Inode = attr.Inode return t } @@ -118,6 +116,9 @@ func EqualEntry(a, b *Entry) bool { if !proto.Equal(a.Remote, b.Remote) { return false } + if a.Quota != b.Quota { + return false + } return true } diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 71ed738f9..0dd7dbee2 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/client/v3" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -82,7 +82,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = weed_util.MaybeGzipData(meta) } @@ -152,7 +152,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u } resp, err := store.client.Get(ctx, string(lastFileStart), - clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) + clientv3.WithFromKey(), clientv3.WithLimit(limit+1)) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } diff --git a/weed/filer/etcd/etcd_store_test.go b/weed/filer/etcd/etcd_store_test.go new file mode 100644 index 000000000..824c28f5a --- /dev/null +++ b/weed/filer/etcd/etcd_store_test.go @@ -0,0 +1,16 @@ +package etcd + +import ( + "github.com/chrislusf/seaweedfs/weed/filer/store_test" + "testing" +) + +func TestStore(t *testing.T) { + // run "make test_etcd" under docker folder. + // to set up local env + if false { + store := &EtcdStore{} + store.initialize("localhost:2379", "3s") + store_test.TestFilerStore(t, store) + } +} diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 00dbf1fd6..4eb657dfa 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -3,11 +3,15 @@ package filer import ( "bytes" "fmt" - "github.com/chrislusf/seaweedfs/weed/wdclient" "io" "math" + "net/url" + "strings" + "sync" "time" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "github.com/golang/protobuf/proto" "github.com/chrislusf/seaweedfs/weed/glog" @@ -19,6 +23,12 @@ const ( ManifestBatch = 10000 ) +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, +} + func HasChunkManifest(chunks []*filer_pb.FileChunk) bool { for _, chunk := range chunks { if chunk.IsChunkManifest { @@ -54,17 +64,17 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk) if err != nil { - return chunks, nil, err + return dataChunks, nil, err } manifestChunks = append(manifestChunks, chunk) // recursive - dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset) + subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset) if subErr != nil { - return chunks, nil, subErr + return dataChunks, nil, subErr } - dataChunks = append(dataChunks, dchunks...) - manifestChunks = append(manifestChunks, mchunks...) + dataChunks = append(dataChunks, subDataChunks...) + manifestChunks = append(manifestChunks, subManifestChunks...) } return } @@ -75,12 +85,15 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c } // IsChunkManifest - data, err := fetchChunk(lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed) + bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer) + bytesBuffer.Reset() + defer bytesBufferPool.Put(bytesBuffer) + err := fetchWholeChunk(bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed) if err != nil { return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err) } m := &filer_pb.FileChunkManifest{} - if err := proto.Unmarshal(data, m); err != nil { + if err := proto.Unmarshal(bytesBuffer.Bytes(), m); err != nil { return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err) } @@ -90,26 +103,43 @@ func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, c } // TODO fetch from cache for weed mount? -func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) { +func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error { urlStrings, err := lookupFileIdFn(fileId) if err != nil { glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) - return nil, err + return err + } + err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0) + if err != nil { + return err } - return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0) + return nil } -func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) { +func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64) (int, error) { + urlStrings, err := lookupFileIdFn(fileId) + if err != nil { + glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) + return 0, err + } + return retriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset) +} + +func retriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64) (n int, err error) { - var err error var shouldRetry bool - receivedData := make([]byte, 0, size) for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 { for _, urlString := range urlStrings { - receivedData = receivedData[:0] - shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { - receivedData = append(receivedData, data...) + n = 0 + if strings.Contains(urlString, "%") { + urlString = url.PathEscape(urlString) + } + shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) { + if n < len(buffer) { + x := copy(buffer[n:], data) + n += x + } }) if !shouldRetry { break @@ -128,7 +158,7 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool } } - return receivedData, err + return n, err } diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index 0dc03f6e2..48b344bf8 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -3,11 +3,12 @@ package filer import ( "bytes" "fmt" - "github.com/chrislusf/seaweedfs/weed/wdclient" "math" - "sort" "sync" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "golang.org/x/exp/slices" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -23,7 +24,16 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) { } func FileSize(entry *filer_pb.Entry) (size uint64) { - return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize) + if entry == nil || entry.Attributes == nil { + return 0 + } + fileSize := entry.Attributes.FileSize + if entry.RemoteEntry != nil { + if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime { + fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize)) + } + } + return maxUint64(TotalSize(entry.Chunks), fileSize) } func ETag(entry *filer_pb.Entry) (etag string) { @@ -101,6 +111,21 @@ func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) { return } +func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) { + + fileIds := make(map[string]bool) + for _, interval := range bs { + fileIds[interval.GetFileIdString()] = true + } + for _, chunk := range as { + if _, found := fileIds[chunk.GetSourceFileId()]; !found { + delta = append(delta, chunk) + } + } + + return +} + type ChunkView struct { FileId string Offset int64 @@ -224,19 +249,26 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) { chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset) + if err != nil { + return + } + + visibles2 := readResolvedChunks(chunks) - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].Mtime == chunks[j].Mtime { - filer_pb.EnsureFid(chunks[i]) - filer_pb.EnsureFid(chunks[j]) - if chunks[i].Fid == nil || chunks[j].Fid == nil { + if true { + return visibles2, err + } + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Mtime == b.Mtime { + filer_pb.EnsureFid(a) + filer_pb.EnsureFid(b) + if a.Fid == nil || b.Fid == nil { return true } - return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey + return a.Fid.FileKey < b.Fid.FileKey } - return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run + return a.Mtime < b.Mtime }) - for _, chunk := range chunks { // glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) @@ -246,9 +278,26 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction } + if len(visibles) != len(visibles2) { + fmt.Printf("different visibles size %d : %d\n", len(visibles), len(visibles2)) + } else { + for i := 0; i < len(visibles); i++ { + checkDifference(visibles[i], visibles2[i]) + } + } + return } +func checkDifference(x, y VisibleInterval) { + if x.start != y.start || + x.stop != y.stop || + x.fileId != y.fileId || + x.modifiedTime != y.modifiedTime { + fmt.Printf("different visible %+v : %+v\n", x, y) + } +} + // find non-overlapping visible intervals // visible interval map to one file chunk diff --git a/weed/filer/filechunks2_test.go b/weed/filer/filechunks2_test.go index 9f9566d9b..39dec87c9 100644 --- a/weed/filer/filechunks2_test.go +++ b/weed/filer/filechunks2_test.go @@ -1,7 +1,7 @@ package filer import ( - "sort" + "golang.org/x/exp/slices" "testing" "github.com/chrislusf/seaweedfs/weed/glog" @@ -34,11 +34,11 @@ func TestCompactFileChunksRealCase(t *testing.T) { } func printChunks(name string, chunks []*filer_pb.FileChunk) { - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].Offset == chunks[j].Offset { - return chunks[i].Mtime < chunks[j].Mtime + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Offset == b.Offset { + return a.Mtime < b.Mtime } - return chunks[i].Offset < chunks[j].Offset + return a.Offset < b.Offset }) for _, chunk := range chunks { glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) diff --git a/weed/filer/filechunks_read.go b/weed/filer/filechunks_read.go new file mode 100644 index 000000000..1d0bd837a --- /dev/null +++ b/weed/filer/filechunks_read.go @@ -0,0 +1,116 @@ +package filer + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "golang.org/x/exp/slices" +) + +func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) { + + var points []*Point + for _, chunk := range chunks { + points = append(points, &Point{ + x: chunk.Offset, + ts: chunk.Mtime, + chunk: chunk, + isStart: true, + }) + points = append(points, &Point{ + x: chunk.Offset + int64(chunk.Size), + ts: chunk.Mtime, + chunk: chunk, + isStart: false, + }) + } + slices.SortFunc(points, func(a, b *Point) bool { + if a.x != b.x { + return a.x < b.x + } + if a.ts != b.ts { + return a.ts < b.ts + } + return !a.isStart + }) + + var prevX int64 + var queue []*Point + for _, point := range points { + if point.isStart { + if len(queue) > 0 { + lastIndex := len(queue) - 1 + lastPoint := queue[lastIndex] + if point.x != prevX && lastPoint.ts < point.ts { + visibles = addToVisibles(visibles, prevX, lastPoint, point) + prevX = point.x + } + } + // insert into queue + for i := len(queue); i >= 0; i-- { + if i == 0 || queue[i-1].ts <= point.ts { + if i == len(queue) { + prevX = point.x + } + queue = addToQueue(queue, i, point) + break + } + } + } else { + lastIndex := len(queue) - 1 + index := lastIndex + var startPoint *Point + for ; index >= 0; index-- { + startPoint = queue[index] + if startPoint.ts == point.ts { + queue = removeFromQueue(queue, index) + break + } + } + if index == lastIndex && startPoint != nil { + visibles = addToVisibles(visibles, prevX, startPoint, point) + prevX = point.x + } + } + } + + return +} + +func removeFromQueue(queue []*Point, index int) []*Point { + for i := index; i < len(queue)-1; i++ { + queue[i] = queue[i+1] + } + queue = queue[:len(queue)-1] + return queue +} + +func addToQueue(queue []*Point, index int, point *Point) []*Point { + queue = append(queue, point) + for i := len(queue) - 1; i > index; i-- { + queue[i], queue[i-1] = queue[i-1], queue[i] + } + return queue +} + +func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval { + if prevX < point.x { + chunk := startPoint.chunk + visibles = append(visibles, VisibleInterval{ + start: prevX, + stop: point.x, + fileId: chunk.GetFileIdString(), + modifiedTime: chunk.Mtime, + chunkOffset: prevX - chunk.Offset, + chunkSize: chunk.Size, + cipherKey: chunk.CipherKey, + isGzipped: chunk.IsCompressed, + }) + } + return visibles +} + +type Point struct { + x int64 + ts int64 + chunk *filer_pb.FileChunk + isStart bool +} diff --git a/weed/filer/filechunks_read_test.go b/weed/filer/filechunks_read_test.go new file mode 100644 index 000000000..e70c66e6f --- /dev/null +++ b/weed/filer/filechunks_read_test.go @@ -0,0 +1,210 @@ +package filer + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "math/rand" + "testing" +) + +func TestReadResolvedChunks(t *testing.T) { + + chunks := []*filer_pb.FileChunk{ + { + FileId: "a", + Offset: 0, + Size: 100, + Mtime: 1, + }, + { + FileId: "b", + Offset: 50, + Size: 100, + Mtime: 2, + }, + { + FileId: "c", + Offset: 200, + Size: 50, + Mtime: 3, + }, + { + FileId: "d", + Offset: 250, + Size: 50, + Mtime: 4, + }, + { + FileId: "e", + Offset: 175, + Size: 100, + Mtime: 5, + }, + } + + visibles := readResolvedChunks(chunks) + + for _, visible := range visibles { + fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime) + } + +} + +func TestRandomizedReadResolvedChunks(t *testing.T) { + + var limit int64 = 1024 * 1024 + array := make([]int64, limit) + var chunks []*filer_pb.FileChunk + for ts := int64(0); ts < 1024; ts++ { + x := rand.Int63n(limit) + y := rand.Int63n(limit) + size := x - y + if size < 0 { + size = -size + } + if size > 1024 { + size = 1024 + } + start := x + if start > y { + start = y + } + chunks = append(chunks, randomWrite(array, start, size, ts)) + } + + visibles := readResolvedChunks(chunks) + + for _, visible := range visibles { + for i := visible.start; i < visible.stop; i++ { + if array[i] != visible.modifiedTime { + t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTime) + } + } + } + + // fmt.Printf("visibles %d", len(visibles)) + +} + +func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.FileChunk { + for i := start; i < start+size; i++ { + array[i] = ts + } + // fmt.Printf("write [%d,%d) %d\n", start, start+size, ts) + return &filer_pb.FileChunk{ + FileId: "", + Offset: start, + Size: uint64(size), + Mtime: ts, + } +} + +func TestSequentialReadResolvedChunks(t *testing.T) { + + var chunkSize int64 = 1024 * 1024 * 2 + var chunks []*filer_pb.FileChunk + for ts := int64(0); ts < 13; ts++ { + chunks = append(chunks, &filer_pb.FileChunk{ + FileId: "", + Offset: chunkSize * ts, + Size: uint64(chunkSize), + Mtime: 1, + }) + } + + visibles := readResolvedChunks(chunks) + + fmt.Printf("visibles %d", len(visibles)) + +} + +func TestActualReadResolvedChunks(t *testing.T) { + + chunks := []*filer_pb.FileChunk{ + { + FileId: "5,e7b96fef48", + Offset: 0, + Size: 2097152, + Mtime: 1634447487595823000, + }, + { + FileId: "5,e5562640b9", + Offset: 2097152, + Size: 2097152, + Mtime: 1634447487595826000, + }, + { + FileId: "5,df033e0fe4", + Offset: 4194304, + Size: 2097152, + Mtime: 1634447487595827000, + }, + { + FileId: "7,eb08148a9b", + Offset: 6291456, + Size: 2097152, + Mtime: 1634447487595827000, + }, + { + FileId: "7,e0f92d1604", + Offset: 8388608, + Size: 2097152, + Mtime: 1634447487595828000, + }, + { + FileId: "7,e33cb63262", + Offset: 10485760, + Size: 2097152, + Mtime: 1634447487595828000, + }, + { + FileId: "5,ea98e40e93", + Offset: 12582912, + Size: 2097152, + Mtime: 1634447487595829000, + }, + { + FileId: "5,e165661172", + Offset: 14680064, + Size: 2097152, + Mtime: 1634447487595829000, + }, + { + FileId: "3,e692097486", + Offset: 16777216, + Size: 2097152, + Mtime: 1634447487595830000, + }, + { + FileId: "3,e28e2e3cbd", + Offset: 18874368, + Size: 2097152, + Mtime: 1634447487595830000, + }, + { + FileId: "3,e443974d4e", + Offset: 20971520, + Size: 2097152, + Mtime: 1634447487595830000, + }, + { + FileId: "2,e815bed597", + Offset: 23068672, + Size: 2097152, + Mtime: 1634447487595831000, + }, + { + FileId: "5,e94715199e", + Offset: 25165824, + Size: 1974736, + Mtime: 1634447487595832000, + }, + } + + visibles := readResolvedChunks(chunks) + + for _, visible := range visibles { + fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTime) + } + +} diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 1a20abefc..86827c50e 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -3,7 +3,11 @@ package filer import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "os" + "sort" "strings" "time" @@ -33,8 +37,6 @@ type Filer struct { fileIdDeletionQueue *util.UnboundedQueue GrpcDialOption grpc.DialOption DirBucketsPath string - FsyncBuckets []string - buckets *FilerBuckets Cipher bool LocalMetaLogBuffer *log_buffer.LogBuffer metaLogCollection string @@ -43,16 +45,18 @@ type Filer struct { Signature int32 FilerConf *FilerConf RemoteStorage *FilerRemoteStorage + UniqueFileId uint32 } -func NewFiler(masters []string, grpcDialOption grpc.DialOption, - filerHost string, filerGrpcPort uint32, collection string, replication string, dataCenter string, notifyFn func()) *Filer { +func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, + filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer { f := &Filer{ - MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, dataCenter, masters), + MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, masters), fileIdDeletionQueue: util.NewUnboundedQueue(), GrpcDialOption: grpcDialOption, FilerConf: NewFilerConf(), RemoteStorage: NewFilerRemoteStorage(), + UniqueFileId: uint32(util.RandomInt32()), } f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn) f.metaLogCollection = collection @@ -63,32 +67,69 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption, return f } -func (f *Filer) AggregateFromPeers(self string, filers []string) { - - // set peers - found := false - for _, peer := range filers { - if peer == self { - found = true - } +func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { + if len(existingNodes) == 0 { + return } - if !found { - filers = append(filers, self) + sort.Slice(existingNodes, func(i, j int) bool { + return existingNodes[i].CreatedAtNs < existingNodes[j].CreatedAtNs + }) + earliestNode := existingNodes[0] + if earliestNode.Address == string(self) { + return } - f.MetaAggregator = NewMetaAggregator(filers, f.GrpcDialOption) - f.MetaAggregator.StartLoopSubscribe(f, self) + glog.V(0).Infof("bootstrap from %v", earliestNode.Address) + err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, "bootstrap", int32(f.UniqueFileId), "/", nil, + 0, snapshotTime.UnixNano(), f.Signature, func(resp *filer_pb.SubscribeMetadataResponse) error { + return Replay(f.Store, resp) + }, pb.FatalOnError) + return +} + +func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) { + + f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption) + f.MasterClient.OnPeerUpdate = f.MetaAggregator.OnPeerUpdate + + for _, peerUpdate := range existingNodes { + f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom) + } } -func (f *Filer) SetStore(store FilerStore) { - f.Store = NewFilerStoreWrapper(store) +func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) { + + if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: f.MasterClient.FilerGroup, + }) + + glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes)) + for _, node := range resp.ClusterNodes { + existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{ + NodeType: cluster.FilerType, + Address: node.Address, + IsLeader: node.IsLeader, + IsAdd: true, + CreatedAtNs: node.CreatedAtNs, + }) + } + return err + }); grpcErr != nil { + glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr) + } + return +} - f.setOrLoadFilerStoreSignature(store) +func (f *Filer) SetStore(store FilerStore) (isFresh bool) { + f.Store = NewFilerStoreWrapper(store) + return f.setOrLoadFilerStoreSignature(store) } -func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) { +func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) (isFresh bool) { storeIdBytes, err := store.KvGet(context.Background(), []byte(FilerStoreId)) if err == ErrKvNotFound || err == nil && len(storeIdBytes) == 0 { f.Signature = util.RandomInt32() @@ -98,23 +139,25 @@ func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) { glog.Fatalf("set %s=%d : %v", FilerStoreId, f.Signature, err) } glog.V(0).Infof("create %s to %d", FilerStoreId, f.Signature) + return true } else if err == nil && len(storeIdBytes) == 4 { f.Signature = int32(util.BytesToUint32(storeIdBytes)) glog.V(0).Infof("existing %s = %d", FilerStoreId, f.Signature) } else { glog.Fatalf("read %v=%v : %v", FilerStoreId, string(storeIdBytes), err) } + return false } func (f *Filer) GetStore() (store FilerStore) { return f.Store } -func (fs *Filer) GetMaster() string { +func (fs *Filer) GetMaster() pb.ServerAddress { return fs.MasterClient.GetMaster() } -func (fs *Filer) KeepConnectedToMaster() { +func (fs *Filer) KeepMasterClientConnected() { fs.MasterClient.KeepConnectedToMaster() } @@ -130,7 +173,7 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error { return f.Store.RollbackTransaction(ctx) } -func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32) error { +func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error { if string(entry.FullPath) == "/" { return nil @@ -148,9 +191,11 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr if oldEntry == nil { - dirParts := strings.Split(string(entry.FullPath), "/") - if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil { - return err + if !skipCreateParentDir { + dirParts := strings.Split(string(entry.FullPath), "/") + if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil { + return err + } } glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name()) @@ -170,7 +215,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr } } - f.maybeAddBucket(entry) f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures) f.deleteChunksIfNotNew(oldEntry, entry) @@ -207,15 +251,13 @@ func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, di dirEntry = &Entry{ FullPath: util.FullPath(dirPath), Attr: Attr{ - Mtime: now, - Crtime: now, - Mode: os.ModeDir | entry.Mode | 0111, - Uid: entry.Uid, - Gid: entry.Gid, - Collection: entry.Collection, - Replication: entry.Replication, - UserName: entry.UserName, - GroupNames: entry.GroupNames, + Mtime: now, + Crtime: now, + Mode: os.ModeDir | entry.Mode | 0111, + Uid: entry.Uid, + Gid: entry.Gid, + UserName: entry.UserName, + GroupNames: entry.GroupNames, }, } @@ -227,7 +269,6 @@ func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, di return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) } } else { - f.maybeAddBucket(dirEntry) f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil) } @@ -285,14 +326,19 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) { lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { - if entry.TtlSec > 0 { - if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - f.Store.DeleteOneEntry(ctx, entry) - expiredCount++ - return true + select { + case <-ctx.Done(): + return false + default: + if entry.TtlSec > 0 { + if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + f.Store.DeleteOneEntry(ctx, entry) + expiredCount++ + return true + } } + return eachEntryFunc(entry) } - return eachEntryFunc(entry) }) if err != nil { return expiredCount, lastFileName, err diff --git a/weed/filer/filer_buckets.go b/weed/filer/filer_buckets.go index 38a1abadb..e9cb3547e 100644 --- a/weed/filer/filer_buckets.go +++ b/weed/filer/filer_buckets.go @@ -1,76 +1,9 @@ package filer import ( - "context" - "math" "strings" - "sync" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/util" ) -type BucketName string -type BucketOption struct { - Name BucketName - Replication string - fsync bool -} -type FilerBuckets struct { - dirBucketsPath string - buckets map[BucketName]*BucketOption - sync.RWMutex -} - -func (f *Filer) LoadBuckets() { - - f.buckets = &FilerBuckets{ - buckets: make(map[BucketName]*BucketOption), - } - - limit := int64(math.MaxInt32) - - entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "", "") - - if err != nil { - glog.V(1).Infof("no buckets found: %v", err) - return - } - - shouldFsyncMap := make(map[string]bool) - for _, bucket := range f.FsyncBuckets { - shouldFsyncMap[bucket] = true - } - - glog.V(1).Infof("buckets found: %d", len(entries)) - - f.buckets.Lock() - for _, entry := range entries { - _, shouldFsnyc := shouldFsyncMap[entry.Name()] - f.buckets.buckets[BucketName(entry.Name())] = &BucketOption{ - Name: BucketName(entry.Name()), - Replication: entry.Replication, - fsync: shouldFsnyc, - } - } - f.buckets.Unlock() - -} - -func (f *Filer) ReadBucketOption(buketName string) (replication string, fsync bool) { - - f.buckets.RLock() - defer f.buckets.RUnlock() - - option, found := f.buckets.buckets[BucketName(buketName)] - - if !found { - return "", false - } - return option.Replication, option.fsync - -} - func (f *Filer) isBucket(entry *Entry) bool { if !entry.IsDirectory() { return false @@ -83,43 +16,6 @@ func (f *Filer) isBucket(entry *Entry) bool { return false } - f.buckets.RLock() - defer f.buckets.RUnlock() - - _, found := f.buckets.buckets[BucketName(dirName)] - - return found - -} - -func (f *Filer) maybeAddBucket(entry *Entry) { - if !entry.IsDirectory() { - return - } - parent, dirName := entry.FullPath.DirAndName() - if parent != f.DirBucketsPath { - return - } - f.addBucket(dirName, &BucketOption{ - Name: BucketName(dirName), - Replication: entry.Replication, - }) -} - -func (f *Filer) addBucket(buketName string, bucketOption *BucketOption) { - - f.buckets.Lock() - defer f.buckets.Unlock() - - f.buckets.buckets[BucketName(buketName)] = bucketOption - -} - -func (f *Filer) deleteBucket(buketName string) { - - f.buckets.Lock() - defer f.buckets.Unlock() - - delete(f.buckets.buckets, BucketName(buketName)) + return true } diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index c58b26dc2..32fc647d9 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -3,6 +3,10 @@ package filer import ( "bytes" "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "google.golang.org/grpc" "io" "github.com/chrislusf/seaweedfs/weed/glog" @@ -26,6 +30,29 @@ type FilerConf struct { rules ptrie.Trie } +func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) { + var buf bytes.Buffer + if err := pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if masterClient != nil { + return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf) + } else { + content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName) + buf = *bytes.NewBuffer(content) + return err + } + }); err != nil && err != filer_pb.ErrNotFound { + return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err) + } + + fc := NewFilerConf() + if buf.Len() > 0 { + if err := fc.LoadFromBytes(buf.Bytes()); err != nil { + return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err) + } + } + return fc, nil +} + func NewFilerConf() (fc *FilerConf) { fc = &FilerConf{ rules: ptrie.New(), @@ -48,12 +75,12 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) { return fc.LoadFromBytes(entry.Content) } - return fc.loadFromChunks(filer, entry.Content, entry.Chunks) + return fc.loadFromChunks(filer, entry.Content, entry.Chunks, entry.Size()) } -func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) { +func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) { if len(content) == 0 { - content, err = filer.readEntry(chunks) + content, err = filer.readEntry(chunks, size) if err != nil { glog.Errorf("read filer conf content: %v", err) return @@ -115,21 +142,32 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf return pathConf } +func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) { + ttls = make(map[string]string) + fc.rules.Walk(func(key []byte, value interface{}) bool { + t := value.(*filer_pb.FilerConf_PathConf) + if t.Collection == collection { + ttls[t.LocationPrefix] = t.GetTtl() + } + return true + }) + return ttls +} + // merge if values in b is not empty, merge them into a func mergePathConf(a, b *filer_pb.FilerConf_PathConf) { a.Collection = util.Nvl(b.Collection, a.Collection) a.Replication = util.Nvl(b.Replication, a.Replication) a.Ttl = util.Nvl(b.Ttl, a.Ttl) - if b.DiskType != "" { - a.DiskType = b.DiskType - } + a.DiskType = util.Nvl(b.DiskType, a.DiskType) a.Fsync = b.Fsync || a.Fsync if b.VolumeGrowthCount > 0 { a.VolumeGrowthCount = b.VolumeGrowthCount } - if b.ReadOnly { - a.ReadOnly = b.ReadOnly - } + a.ReadOnly = b.ReadOnly || a.ReadOnly + a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter) + a.Rack = util.Nvl(b.Rack, a.Rack) + a.DataNode = util.Nvl(b.DataNode, a.DataNode) } func (fc *FilerConf) ToProto() *filer_pb.FilerConf { diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 35187d034..0fc2f6c3c 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -9,12 +9,13 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -type HardLinkId []byte - const ( MsgFailDelNonEmptyFolder = "fail to delete non-empty folder" ) +type OnChunksFunc func([]*filer_pb.FileChunk) error +type OnHardLinkIdsFunc func([]HardLinkId) error + func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) { if p == "/" { return nil @@ -24,23 +25,30 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if findErr != nil { return findErr } - isDeleteCollection := f.isBucket(entry) - - var chunks []*filer_pb.FileChunk - var hardLinkIds []HardLinkId - chunks = append(chunks, entry.Chunks...) if entry.IsDirectory() { // delete the folder children, not including the folder itself - var dirChunks []*filer_pb.FileChunk - var dirHardLinkIds []HardLinkId - dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures) + err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error { + if shouldDeleteChunks && !isDeleteCollection { + f.DirectDeleteChunks(chunks) + } + return nil + }, func(hardLinkIds []HardLinkId) error { + // A case not handled: + // what if the chunk is in a different collection? + if shouldDeleteChunks { + f.maybeDeleteHardLinks(hardLinkIds) + } + return nil + }) if err != nil { glog.V(0).Infof("delete directory %s: %v", p, err) return fmt.Errorf("delete directory %s: %v", p, err) } - chunks = append(chunks, dirChunks...) - hardLinkIds = append(hardLinkIds, dirHardLinkIds...) + } + + if shouldDeleteChunks && !isDeleteCollection { + f.DirectDeleteChunks(entry.Chunks) } // delete the file or folder @@ -49,25 +57,15 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR return fmt.Errorf("delete file %s: %v", p, err) } - if shouldDeleteChunks && !isDeleteCollection { - f.DirectDeleteChunks(chunks) - } - // A case not handled: - // what if the chunk is in a different collection? - if shouldDeleteChunks { - f.maybeDeleteHardLinks(hardLinkIds) - } - if isDeleteCollection { collectionName := entry.Name() f.doDeleteCollection(collectionName) - f.deleteBucket(collectionName) } return nil } -func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32) (chunks []*filer_pb.FileChunk, hardlinkIds []HardLinkId, err error) { +func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onChunksFn OnChunksFunc, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) { lastFileName := "" includeLastFile := false @@ -76,34 +74,30 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "") if err != nil { glog.Errorf("list folder %s: %v", entry.FullPath, err) - return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) + return fmt.Errorf("list folder %s: %v", entry.FullPath, err) } if lastFileName == "" && !isRecursive && len(entries) > 0 { // only for first iteration in the loop - glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name()) - return nil, nil, fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath) + glog.V(0).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name()) + return fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath) } for _, sub := range entries { lastFileName = sub.Name() - var dirChunks []*filer_pb.FileChunk - var dirHardLinkIds []HardLinkId if sub.IsDirectory() { subIsDeletingBucket := f.isBucket(sub) - dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil) - chunks = append(chunks, dirChunks...) - hardlinkIds = append(hardlinkIds, dirHardLinkIds...) + err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil, onChunksFn, onHardLinkIdsFn) } else { f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil) if len(sub.HardLinkId) != 0 { // hard link chunk data are deleted separately - hardlinkIds = append(hardlinkIds, sub.HardLinkId) + err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId}) } else { - chunks = append(chunks, sub.Chunks...) + err = onChunksFn(sub.Chunks) } } if err != nil && !ignoreRecursiveError { - return nil, nil, err + return err } } @@ -113,22 +107,26 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry } } - glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks) + glog.V(3).Infof("deleting directory %v delete chunks: %v", entry.FullPath, shouldDeleteChunks) if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil { - return nil, nil, fmt.Errorf("filer store delete: %v", storeDeletionErr) + return fmt.Errorf("filer store delete: %v", storeDeletionErr) } f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures) - return chunks, hardlinkIds, nil + return nil } func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool, isFromOtherCluster bool, signatures []int32) (err error) { glog.V(3).Infof("deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks) - if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil { + if !entry.IsDirectory() && !shouldDeleteChunks { + if storeDeletionErr := f.Store.DeleteOneEntrySkipHardlink(ctx, entry.FullPath); storeDeletionErr != nil { + return fmt.Errorf("filer store delete skip hardlink: %v", storeDeletionErr) + } + } else if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil { return fmt.Errorf("filer store delete: %v", storeDeletionErr) } if !entry.IsDirectory() { @@ -140,7 +138,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou func (f *Filer) doDeleteCollection(collectionName string) (err error) { - return f.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + return f.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ Name: collectionName, }) diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index a12587541..e73f94151 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -1,6 +1,7 @@ package filer import ( + "math" "strings" "time" @@ -129,6 +130,12 @@ func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) { } } +func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) { + for _, chunk := range chunks { + f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString()) + } +} + func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) { if oldEntry == nil { @@ -136,18 +143,41 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) { } if newEntry == nil { f.DeleteChunks(oldEntry.Chunks) + return } var toDelete []*filer_pb.FileChunk newChunkIds := make(map[string]bool) - for _, newChunk := range newEntry.Chunks { + newDataChunks, newManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + newEntry.Chunks, 0, math.MaxInt64) + if err != nil { + glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s", + newEntry.Chunks, oldEntry.Chunks) + return + } + for _, newChunk := range newDataChunks { + newChunkIds[newChunk.GetFileIdString()] = true + } + for _, newChunk := range newManifestChunks { newChunkIds[newChunk.GetFileIdString()] = true } - for _, oldChunk := range oldEntry.Chunks { + oldDataChunks, oldManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + oldEntry.Chunks, 0, math.MaxInt64) + if err != nil { + glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s", + newEntry.Chunks, oldEntry.Chunks) + return + } + for _, oldChunk := range oldDataChunks { + if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { + toDelete = append(toDelete, oldChunk) + } + } + for _, oldChunk := range oldManifestChunks { if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { toDelete = append(toDelete, oldChunk) } } - f.DeleteChunks(toDelete) + f.DeleteChunksNotRecursive(toDelete) } diff --git a/weed/filer/filer_hardlink.go b/weed/filer/filer_hardlink.go new file mode 100644 index 000000000..7a91602fd --- /dev/null +++ b/weed/filer/filer_hardlink.go @@ -0,0 +1,16 @@ +package filer + +import ( + "github.com/chrislusf/seaweedfs/weed/util" +) + +const ( + HARD_LINK_MARKER = '\x01' +) + +type HardLinkId []byte // 16 bytes + 1 marker byte + +func NewHardLinkId() HardLinkId { + bytes := append(util.RandomBytes(16), HARD_LINK_MARKER) + return bytes +} diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index 7ab101102..4d26a695c 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "math" "strings" "time" @@ -92,14 +93,14 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) { startTime, stopTime = startTime.UTC(), stopTime.UTC() - targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir, - startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), + targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir, + startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFileId, // startTime.Second(), startTime.Nanosecond(), ) for { if err := f.appendToFile(targetFile, buf); err != nil { - glog.V(1).Infof("log write failed %s: %v", targetFile, err) + glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err) time.Sleep(737 * time.Millisecond) } else { break @@ -107,49 +108,67 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) { } } -func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) { +func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, isDone bool, err error) { startTime = startTime.UTC() startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day()) - startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute()) + startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute()) + var stopDate, stopHourMinute string + if stopTsNs != 0 { + stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Nanosecond)).UTC() + stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day()) + stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute()) + } sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "", "") + dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "") if listDayErr != nil { - return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) + return lastTsNs, isDone, fmt.Errorf("fail to list log by day: %v", listDayErr) } for _, dayEntry := range dayEntries { + if stopDate != "" { + if strings.Compare(dayEntry.Name(), stopDate) > 0 { + break + } + } // println("checking day", dayEntry.FullPath) - hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "", "") + hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "") if listHourMinuteErr != nil { - return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) + return lastTsNs, isDone, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) } for _, hourMinuteEntry := range hourMinuteEntries { // println("checking hh-mm", hourMinuteEntry.FullPath) if dayEntry.Name() == startDate { - if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 { + hourMinute := util.FileNameBase(hourMinuteEntry.Name()) + if strings.Compare(hourMinute, startHourMinute) < 0 { continue } } + if dayEntry.Name() == stopDate { + hourMinute := util.FileNameBase(hourMinuteEntry.Name()) + if strings.Compare(hourMinute, stopHourMinute) > 0 { + break + } + } // println("processing", hourMinuteEntry.FullPath) chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks) - if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { + if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, stopTsNs, eachLogEntryFn); err != nil { chunkedFileReader.Close() if err == io.EOF { continue } - return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err) + return lastTsNs, isDone, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err) } chunkedFileReader.Close() } } - return lastTsNs, nil + return lastTsNs, isDone, nil } -func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) { +func ReadEachLogEntry(r io.Reader, sizeBuf []byte, startTsNs, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) { for { n, err := r.Read(sizeBuf) if err != nil { @@ -172,9 +191,12 @@ func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func if err = proto.Unmarshal(entryData, logEntry); err != nil { return lastTsNs, err } - if logEntry.TsNs <= ns { + if logEntry.TsNs <= startTsNs { continue } + if stopTsNs != 0 && logEntry.TsNs > stopTsNs { + return lastTsNs, err + } // println("each log: ", logEntry.TsNs) if err := eachLogEntryFn(logEntry); err != nil { return lastTsNs, err diff --git a/weed/filer/filer_notify_append.go b/weed/filer/filer_notify_append.go index d441bbbc9..25b99d0f7 100644 --- a/weed/filer/filer_notify_append.go +++ b/weed/filer/filer_notify_append.go @@ -33,6 +33,8 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error { Gid: OS_GID, }, } + } else if err != nil { + return fmt.Errorf("find %s: %v", fullpath, err) } else { offset = int64(TotalSize(entry.Chunks)) } @@ -41,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error { entry.Chunks = append(entry.Chunks, uploadResult.ToPbFileChunk(assignResult.Fid, offset)) // update the entry - err = f.CreateEntry(context.Background(), entry, false, false, nil) + err = f.CreateEntry(context.Background(), entry, false, false, nil, false) return err } @@ -66,7 +68,16 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi // upload data targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid - uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth) + uploadOption := &operation.UploadOption{ + UploadUrl: targetUrl, + Filename: "", + Cipher: f.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: assignResult.Auth, + } + uploadResult, err := operation.UploadData(data, uploadOption) if err != nil { return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) } diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index 34ac5321a..3b290deca 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -2,8 +2,6 @@ package filer import ( "bytes" - "math" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -24,12 +22,12 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) { } } if f.DirBucketsPath == event.Directory { - if message.OldEntry == nil && message.NewEntry != nil { + if filer_pb.IsCreate(event) { if message.NewEntry.IsDirectory { f.Store.OnBucketCreation(message.NewEntry.Name) } } - if message.OldEntry != nil && message.NewEntry == nil { + if filer_pb.IsDelete(event) { if message.OldEntry.IsDirectory { f.Store.OnBucketDeletion(message.OldEntry.Name) } @@ -55,9 +53,9 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR } } -func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { +func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) { var buf bytes.Buffer - err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) + err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size)) if err != nil { return nil, err } @@ -66,7 +64,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { fc := NewFilerConf() - err := fc.loadFromChunks(f, entry.Content, entry.Chunks) + err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry)) if err != nil { glog.Errorf("read filer conf chunks: %v", err) return diff --git a/weed/filer/filer_search.go b/weed/filer/filer_search.go index 2e0336da8..112df7984 100644 --- a/weed/filer/filer_search.go +++ b/weed/filer/filer_search.go @@ -23,15 +23,15 @@ func splitPattern(pattern string) (prefix string, restPattern string) { // For now, prefix and namePattern are mutually exclusive func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) { + if limit > math.MaxInt32-1 { + limit = math.MaxInt32 - 1 + } + _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool { entries = append(entries, entry) return true }) - if limit == math.MaxInt64 { - limit = math.MaxInt64 - 1 - } - hasMore = int64(len(entries)) >= limit+1 if hasMore { entries = entries[:limit] diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index 38927d6fb..260945b33 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -4,8 +4,11 @@ import ( "context" "errors" "github.com/chrislusf/seaweedfs/weed/util" + "io" ) +const CountEntryChunksForGzip = 50 + var ( ErrUnsupportedListDirectoryPrefixed = errors.New("unsupported directory prefix listing") ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing") @@ -45,3 +48,7 @@ type BucketAware interface { OnBucketDeletion(bucket string) CanDropWholeBucket() bool } + +type Debuggable interface { + Debug(writer io.Writer) +} diff --git a/weed/filer/filerstore_hardlink.go b/weed/filer/filerstore_hardlink.go index 316c76a0c..ae2f5fee7 100644 --- a/weed/filer/filerstore_hardlink.go +++ b/weed/filer/filerstore_hardlink.go @@ -9,16 +9,20 @@ import ( ) func (fsw *FilerStoreWrapper) handleUpdateToHardLinks(ctx context.Context, entry *Entry) error { - if len(entry.HardLinkId) == 0 { + + if entry.IsDirectory() { return nil } - // handle hard links - if err := fsw.setHardLink(ctx, entry); err != nil { - return fmt.Errorf("setHardLink %d: %v", entry.HardLinkId, err) + + if len(entry.HardLinkId) > 0 { + // handle hard links + if err := fsw.setHardLink(ctx, entry); err != nil { + return fmt.Errorf("setHardLink %d: %v", entry.HardLinkId, err) + } } // check what is existing entry - glog.V(4).Infof("handleUpdateToHardLinks FindEntry %s", entry.FullPath) + // glog.V(4).Infof("handleUpdateToHardLinks FindEntry %s", entry.FullPath) actualStore := fsw.getActualStore(entry.FullPath) existingEntry, err := actualStore.FindEntry(ctx, entry.FullPath) if err != nil && err != filer_pb.ErrNotFound { @@ -46,6 +50,8 @@ func (fsw *FilerStoreWrapper) setHardLink(ctx context.Context, entry *Entry) err return encodeErr } + glog.V(4).Infof("setHardLink %v nlink:%d", entry.FullPath, entry.HardLinkCounter) + return fsw.KvPut(ctx, key, newBlob) } @@ -55,7 +61,6 @@ func (fsw *FilerStoreWrapper) maybeReadHardLink(ctx context.Context, entry *Entr } key := entry.HardLinkId - glog.V(4).Infof("maybeReadHardLink KvGet %v", key) value, err := fsw.KvGet(ctx, key) if err != nil { glog.Errorf("read %s hardlink %d: %v", entry.FullPath, entry.HardLinkId, err) @@ -67,6 +72,8 @@ func (fsw *FilerStoreWrapper) maybeReadHardLink(ctx context.Context, entry *Entr return err } + glog.V(4).Infof("maybeReadHardLink %v nlink:%d", entry.FullPath, entry.HardLinkCounter) + return nil } diff --git a/weed/filer/filerstore_translate_path.go b/weed/filer/filerstore_translate_path.go index 00bf82ed4..9e74dd41c 100644 --- a/weed/filer/filerstore_translate_path.go +++ b/weed/filer/filerstore_translate_path.go @@ -3,20 +3,21 @@ package filer import ( "context" "github.com/chrislusf/seaweedfs/weed/util" + "math" "strings" ) var ( - _ = FilerStore(&FilerStorePathTranlator{}) + _ = FilerStore(&FilerStorePathTranslator{}) ) -type FilerStorePathTranlator struct { +type FilerStorePathTranslator struct { actualStore FilerStore storeRoot string } -func NewFilerStorePathTranlator(storeRoot string, store FilerStore) *FilerStorePathTranlator { - if innerStore, ok := store.(*FilerStorePathTranlator); ok { +func NewFilerStorePathTranslator(storeRoot string, store FilerStore) *FilerStorePathTranslator { + if innerStore, ok := store.(*FilerStorePathTranslator); ok { return innerStore } @@ -24,13 +25,13 @@ func NewFilerStorePathTranlator(storeRoot string, store FilerStore) *FilerStoreP storeRoot += "/" } - return &FilerStorePathTranlator{ + return &FilerStorePathTranslator{ actualStore: store, storeRoot: storeRoot, } } -func (t *FilerStorePathTranlator) translatePath(fp util.FullPath) (newPath util.FullPath) { +func (t *FilerStorePathTranslator) translatePath(fp util.FullPath) (newPath util.FullPath) { newPath = fp if t.storeRoot == "/" { return @@ -41,7 +42,7 @@ func (t *FilerStorePathTranlator) translatePath(fp util.FullPath) (newPath util. } return } -func (t *FilerStorePathTranlator) changeEntryPath(entry *Entry) (previousPath util.FullPath) { +func (t *FilerStorePathTranslator) changeEntryPath(entry *Entry) (previousPath util.FullPath) { previousPath = entry.FullPath if t.storeRoot == "/" { return @@ -49,33 +50,33 @@ func (t *FilerStorePathTranlator) changeEntryPath(entry *Entry) (previousPath ut entry.FullPath = t.translatePath(previousPath) return } -func (t *FilerStorePathTranlator) recoverEntryPath(entry *Entry, previousPath util.FullPath) { +func (t *FilerStorePathTranslator) recoverEntryPath(entry *Entry, previousPath util.FullPath) { entry.FullPath = previousPath } -func (t *FilerStorePathTranlator) GetName() string { +func (t *FilerStorePathTranslator) GetName() string { return t.actualStore.GetName() } -func (t *FilerStorePathTranlator) Initialize(configuration util.Configuration, prefix string) error { +func (t *FilerStorePathTranslator) Initialize(configuration util.Configuration, prefix string) error { return t.actualStore.Initialize(configuration, prefix) } -func (t *FilerStorePathTranlator) InsertEntry(ctx context.Context, entry *Entry) error { +func (t *FilerStorePathTranslator) InsertEntry(ctx context.Context, entry *Entry) error { previousPath := t.changeEntryPath(entry) defer t.recoverEntryPath(entry, previousPath) return t.actualStore.InsertEntry(ctx, entry) } -func (t *FilerStorePathTranlator) UpdateEntry(ctx context.Context, entry *Entry) error { +func (t *FilerStorePathTranslator) UpdateEntry(ctx context.Context, entry *Entry) error { previousPath := t.changeEntryPath(entry) defer t.recoverEntryPath(entry, previousPath) return t.actualStore.UpdateEntry(ctx, entry) } -func (t *FilerStorePathTranlator) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error) { +func (t *FilerStorePathTranslator) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error) { if t.storeRoot == "/" { return t.actualStore.FindEntry(ctx, fp) } @@ -87,12 +88,12 @@ func (t *FilerStorePathTranlator) FindEntry(ctx context.Context, fp util.FullPat return } -func (t *FilerStorePathTranlator) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) { +func (t *FilerStorePathTranslator) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) { newFullPath := t.translatePath(fp) return t.actualStore.DeleteEntry(ctx, newFullPath) } -func (t *FilerStorePathTranlator) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) { +func (t *FilerStorePathTranslator) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) { previousPath := t.changeEntryPath(existingEntry) defer t.recoverEntryPath(existingEntry, previousPath) @@ -100,13 +101,13 @@ func (t *FilerStorePathTranlator) DeleteOneEntry(ctx context.Context, existingEn return t.actualStore.DeleteEntry(ctx, existingEntry.FullPath) } -func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { +func (t *FilerStorePathTranslator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { newFullPath := t.translatePath(fp) return t.actualStore.DeleteFolderChildren(ctx, newFullPath) } -func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) { +func (t *FilerStorePathTranslator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) { newFullPath := t.translatePath(dirPath) @@ -116,38 +117,42 @@ func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirP }) } -func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) { +func (t *FilerStorePathTranslator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) { newFullPath := t.translatePath(dirPath) + if limit > math.MaxInt32-1 { + limit = math.MaxInt32 - 1 + } + return t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool { entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath return eachEntryFunc(entry) }) } -func (t *FilerStorePathTranlator) BeginTransaction(ctx context.Context) (context.Context, error) { +func (t *FilerStorePathTranslator) BeginTransaction(ctx context.Context) (context.Context, error) { return t.actualStore.BeginTransaction(ctx) } -func (t *FilerStorePathTranlator) CommitTransaction(ctx context.Context) error { +func (t *FilerStorePathTranslator) CommitTransaction(ctx context.Context) error { return t.actualStore.CommitTransaction(ctx) } -func (t *FilerStorePathTranlator) RollbackTransaction(ctx context.Context) error { +func (t *FilerStorePathTranslator) RollbackTransaction(ctx context.Context) error { return t.actualStore.RollbackTransaction(ctx) } -func (t *FilerStorePathTranlator) Shutdown() { +func (t *FilerStorePathTranslator) Shutdown() { t.actualStore.Shutdown() } -func (t *FilerStorePathTranlator) KvPut(ctx context.Context, key []byte, value []byte) (err error) { +func (t *FilerStorePathTranslator) KvPut(ctx context.Context, key []byte, value []byte) (err error) { return t.actualStore.KvPut(ctx, key, value) } -func (t *FilerStorePathTranlator) KvGet(ctx context.Context, key []byte) (value []byte, err error) { +func (t *FilerStorePathTranslator) KvGet(ctx context.Context, key []byte) (value []byte, err error) { return t.actualStore.KvGet(ctx, key) } -func (t *FilerStorePathTranlator) KvDelete(ctx context.Context, key []byte) (err error) { +func (t *FilerStorePathTranslator) KvDelete(ctx context.Context, key []byte) (err error) { return t.actualStore.KvDelete(ctx, key) } diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 2470f340c..3ece25ce6 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -4,6 +4,8 @@ import ( "context" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/viant/ptrie" + "io" + "math" "strings" "time" @@ -14,12 +16,14 @@ import ( var ( _ = VirtualFilerStore(&FilerStoreWrapper{}) + _ = Debuggable(&FilerStoreWrapper{}) ) type VirtualFilerStore interface { FilerStore DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error DeleteOneEntry(ctx context.Context, entry *Entry) error + DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) error AddPathSpecificStore(path string, storeId string, store FilerStore) OnBucketCreation(bucket string) OnBucketDeletion(bucket string) @@ -72,7 +76,7 @@ func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) { } func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) { - fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store) + fsw.storeIdToStore[storeId] = NewFilerStorePathTranslator(path, store) err := fsw.pathToStore.Put([]byte(path), storeId) if err != nil { glog.Fatalf("put path specific store: %v", err) @@ -124,7 +128,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err return err } - glog.V(4).Infof("InsertEntry %s", entry.FullPath) + // glog.V(4).Infof("InsertEntry %s", entry.FullPath) return actualStore.InsertEntry(ctx, entry) } @@ -145,7 +149,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err return err } - glog.V(4).Infof("UpdateEntry %s", entry.FullPath) + // glog.V(4).Infof("UpdateEntry %s", entry.FullPath) return actualStore.UpdateEntry(ctx, entry) } @@ -189,7 +193,7 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) } } - glog.V(4).Infof("DeleteEntry %s", fp) + // glog.V(4).Infof("DeleteEntry %s", fp) return actualStore.DeleteEntry(ctx, fp) } @@ -209,10 +213,22 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry } } - glog.V(4).Infof("DeleteOneEntry %s", existingEntry.FullPath) + // glog.V(4).Infof("DeleteOneEntry %s", existingEntry.FullPath) return actualStore.DeleteEntry(ctx, existingEntry.FullPath) } +func (fsw *FilerStoreWrapper) DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) (err error) { + actualStore := fsw.getActualStore(fullpath) + stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc() + start := time.Now() + defer func() { + stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "delete").Observe(time.Since(start).Seconds()) + }() + + glog.V(4).Infof("DeleteOneEntrySkipHardlink %s", fullpath) + return actualStore.DeleteEntry(ctx, fullpath) +} + func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { actualStore := fsw.getActualStore(fp + "/") stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc() @@ -221,7 +237,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds()) }() - glog.V(4).Infof("DeleteFolderChildren %s", fp) + // glog.V(4).Infof("DeleteFolderChildren %s", fp) return actualStore.DeleteFolderChildren(ctx, fp) } @@ -233,7 +249,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() - glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit) + // glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit) return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) bool { fsw.maybeReadHardLink(ctx, entry) filer_pb.AfterEntryDeserialization(entry.Chunks) @@ -248,14 +264,18 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, defer func() { stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds()) }() - glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit) - lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, eachEntryFunc) + if limit > math.MaxInt32-1 { + limit = math.MaxInt32 - 1 + } + // glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit) + adjustedEntryFunc := func(entry *Entry) bool { + fsw.maybeReadHardLink(ctx, entry) + filer_pb.AfterEntryDeserialization(entry.Chunks) + return eachEntryFunc(entry) + } + lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc) if err == ErrUnsupportedListDirectoryPrefixed { - lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool { - fsw.maybeReadHardLink(ctx, entry) - filer_pb.AfterEntryDeserialization(entry.Chunks) - return eachEntryFunc(entry) - }) + lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc) } return lastFileName, err } @@ -278,8 +298,10 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u count := int64(0) for count < limit && len(notPrefixed) > 0 { + var isLastItemHasPrefix bool for _, entry := range notPrefixed { if strings.HasPrefix(entry.Name(), prefix) { + isLastItemHasPrefix = true count++ if !eachEntryFunc(entry) { return @@ -287,17 +309,21 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u if count >= limit { break } + } else { + isLastItemHasPrefix = false } } - if count < limit { + if count < limit && isLastItemHasPrefix && len(notPrefixed) == int(limit) { notPrefixed = notPrefixed[:0] - _, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { + lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry) return true }) if err != nil { return } + } else { + break } } return @@ -328,3 +354,9 @@ func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []by func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) { return fsw.getDefaultStore().KvDelete(ctx, key) } + +func (fsw *FilerStoreWrapper) Debug(writer io.Writer) { + if debuggable, ok := fsw.getDefaultStore().(Debuggable); ok { + debuggable.Debug(writer) + } +} diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index e0d878ca7..c5d6eb48c 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -75,7 +75,7 @@ func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) er if err != nil { return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index ce454f36a..6abb37f99 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -6,8 +6,10 @@ import ( "fmt" "github.com/syndtr/goleveldb/leveldb" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "io" "os" "github.com/chrislusf/seaweedfs/weed/filer" @@ -20,6 +22,10 @@ const ( DIR_FILE_SEPARATOR = byte(0x00) ) +var ( + _ = filer.Debuggable(&LevelDBStore{}) +) + func init() { filer.Stores = append(filer.Stores, &LevelDBStore{}) } @@ -45,9 +51,9 @@ func (store *LevelDBStore) initialize(dir string) (err error) { } opts := &opt.Options{ - BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB - CompactionTableSizeMultiplier: 10, + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + Filter: filter.NewBloomFilter(8), // false positive rate 0.02 } if store.db, err = leveldb.OpenFile(dir, opts); err != nil { @@ -80,7 +86,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } @@ -241,3 +247,13 @@ func getNameFromKey(key []byte) string { func (store *LevelDBStore) Shutdown() { store.db.Close() } + +func (store *LevelDBStore) Debug(writer io.Writer) { + iter := store.db.NewIterator(&leveldb_util.Range{}, nil) + for iter.Next() { + key := iter.Key() + fullName := bytes.Replace(key, []byte{DIR_FILE_SEPARATOR}, []byte{' '}, 1) + fmt.Fprintf(writer, "%v\n", string(fullName)) + } + iter.Release() +} diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index d437895f5..4cd8b88e8 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -3,7 +3,6 @@ package leveldb import ( "context" "fmt" - "io/ioutil" "os" "testing" "time" @@ -13,9 +12,8 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -33,7 +31,7 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } @@ -67,9 +65,8 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -90,9 +87,8 @@ func TestEmptyRoot(t *testing.T) { } func BenchmarkInsertEntry(b *testing.B) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := b.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 4c4409c4d..d68493bd7 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -46,10 +46,9 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { } opts := &opt.Options{ - BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB - CompactionTableSizeMultiplier: 4, - Filter: filter.NewBloomFilter(8), // false positive rate 0.02 + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + Filter: filter.NewBloomFilter(8), // false positive rate 0.02 } for d := 0; d < dbCount; d++ { @@ -89,7 +88,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index fd0ad18a3..1f8e33116 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -2,8 +2,6 @@ package leveldb import ( "context" - "io/ioutil" - "os" "testing" "github.com/chrislusf/seaweedfs/weed/filer" @@ -11,9 +9,8 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) testFiler.SetStore(store) @@ -31,7 +28,7 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } @@ -65,9 +62,8 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) testFiler.SetStore(store) diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index bc57a6605..d21515bd4 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -66,17 +66,15 @@ func (store *LevelDB3Store) initialize(dir string) (err error) { func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { bloom := filter.NewBloomFilter(8) // false positive rate 0.02 opts := &opt.Options{ - BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB - CompactionTableSizeMultiplier: 4, - Filter: bloom, + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + Filter: bloom, } if name != DEFAULT { opts = &opt.Options{ - BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB - CompactionTableSizeMultiplier: 4, - Filter: bloom, + BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB + Filter: bloom, } } @@ -179,7 +177,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index 0b970a539..823c3a1bf 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -2,8 +2,6 @@ package leveldb import ( "context" - "io/ioutil" - "os" "testing" "github.com/chrislusf/seaweedfs/weed/filer" @@ -11,9 +9,8 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) testFiler.SetStore(store) @@ -31,7 +28,7 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } @@ -65,9 +62,8 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index 913cbd454..fb96ee01b 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -3,6 +3,8 @@ package filer import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/util" "io" "sync" @@ -18,9 +20,13 @@ import ( ) type MetaAggregator struct { - filers []string - grpcDialOption grpc.DialOption - MetaLogBuffer *log_buffer.LogBuffer + filer *Filer + self pb.ServerAddress + isLeader bool + grpcDialOption grpc.DialOption + MetaLogBuffer *log_buffer.LogBuffer + peerStatues map[pb.ServerAddress]int + peerStatuesLock sync.Mutex // notifying clients ListenersLock sync.Mutex ListenersCond *sync.Cond @@ -28,10 +34,12 @@ type MetaAggregator struct { // MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk. // The old data comes from what each LocalMetadata persisted on disk. -func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator { +func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator { t := &MetaAggregator{ - filers: filers, + filer: filer, + self: self, grpcDialOption: grpcDialOption, + peerStatues: make(map[pb.ServerAddress]int), } t.ListenersCond = sync.NewCond(&t.ListenersLock) t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() { @@ -40,13 +48,66 @@ func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAgg return t } -func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) { - for _, filer := range ma.filers { - go ma.subscribeToOneFiler(f, self, filer) +func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) { + if update.NodeType != cluster.FilerType { + return } + + address := pb.ServerAddress(update.Address) + if update.IsAdd { + // every filer should subscribe to a new filer + if ma.setActive(address, true) { + go ma.loopSubscribeToOnefiler(ma.filer, ma.self, address, startFrom) + } + } else { + ma.setActive(address, false) + } +} + +func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) { + ma.peerStatuesLock.Lock() + defer ma.peerStatuesLock.Unlock() + if isActive { + if _, found := ma.peerStatues[address]; found { + ma.peerStatues[address] += 1 + } else { + ma.peerStatues[address] = 1 + notDuplicated = true + } + } else { + if _, found := ma.peerStatues[address]; found { + delete(ma.peerStatues, address) + } + } + return +} +func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) { + ma.peerStatuesLock.Lock() + defer ma.peerStatuesLock.Unlock() + var count int + count, isActive = ma.peerStatues[address] + return count > 0 && isActive } -func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) { +func (ma *MetaAggregator) loopSubscribeToOnefiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time) { + lastTsNs := startFrom.UnixNano() + for { + glog.V(0).Infof("loopSubscribeToOnefiler read %s start from %v %d", peer, time.Unix(0, lastTsNs), lastTsNs) + nextLastTsNs, err := ma.doSubscribeToOneFiler(f, self, peer, lastTsNs) + if !ma.isActive(peer) { + glog.V(0).Infof("stop subscribing remote %s meta change", peer) + return + } + if err != nil { + glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err) + } else if lastTsNs < nextLastTsNs { + lastTsNs = nextLastTsNs + } + time.Sleep(1733 * time.Millisecond) + } +} + +func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom int64) (int64, error) { /* Each filer reads the "filer.store.id", which is the store's signature when filer starts. @@ -60,20 +121,26 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse) lastPersistTime := time.Now() - lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano() + lastTsNs := startFrom peerSignature, err := ma.readFilerStoreSignature(peer) - for err != nil { - glog.V(0).Infof("connecting to peer filer %s: %v", peer, err) - time.Sleep(1357 * time.Millisecond) - peerSignature, err = ma.readFilerStoreSignature(peer) + if err != nil { + return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err) } // when filer store is not shared by multiple filers if peerSignature != f.Signature { - lastTsNs = 0 if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil { lastTsNs = prevTsNs + defer func(prevTsNs int64) { + if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() { + if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil { + glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs) + } else { + glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs) + } + } + }(prevTsNs) } glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs) @@ -110,54 +177,50 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string } dir := event.Directory // println("received meta change", dir, "size", len(data)) - ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, 0) + ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs) if maybeReplicateMetadataChange != nil { maybeReplicateMetadataChange(event) } return nil } - for { - glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs)) - err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ - ClientName: "filer:" + self, - PathPrefix: "/", - SinceNs: lastTsNs, - }) - if err != nil { - return fmt.Errorf("subscribe: %v", err) - } + glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs)) + err = pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ + ClientName: "filer:" + string(self), + PathPrefix: "/", + SinceNs: lastTsNs, + ClientId: int32(ma.filer.UniqueFileId), + }) + if err != nil { + return fmt.Errorf("subscribe: %v", err) + } - for { - resp, listenErr := stream.Recv() - if listenErr == io.EOF { - return nil - } - if listenErr != nil { - return listenErr - } + for { + resp, listenErr := stream.Recv() + if listenErr == io.EOF { + return nil + } + if listenErr != nil { + return listenErr + } - if err := processEventFn(resp); err != nil { - return fmt.Errorf("process %v: %v", resp, err) - } - lastTsNs = resp.TsNs + if err := processEventFn(resp); err != nil { + return fmt.Errorf("process %v: %v", resp, err) + } + lastTsNs = resp.TsNs - f.onMetadataChangeEvent(resp) + f.onMetadataChangeEvent(resp) - } - }) - if err != nil { - glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err) - time.Sleep(1733 * time.Millisecond) } - } + }) + return lastTsNs, err } -func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) { - err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) { + err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return err @@ -172,18 +235,13 @@ const ( MetaOffsetPrefix = "Meta" ) -func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) { +func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) { key := []byte(MetaOffsetPrefix + "xxxx") util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature)) value, err := f.Store.KvGet(context.Background(), key) - if err == ErrKvNotFound { - glog.Warningf("readOffset %s not found", peer) - return 0, nil - } - if err != nil { return 0, fmt.Errorf("readOffset %s : %v", peer, err) } @@ -195,7 +253,7 @@ func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) return } -func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) { +func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) { key := []byte(MetaOffsetPrefix + "xxxx") util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature)) diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 1ef5056f4..83686bfe7 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -107,7 +107,7 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } @@ -159,7 +159,7 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa dir, name := fullpath.DirAndName() where := bson.M{"directory": dir, "name": name} - _, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteOne(ctx, where) + _, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -193,11 +193,15 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti optLimit := int64(limit) opts := &options.FindOptions{Limit: &optLimit, Sort: bson.M{"name": 1}} cur, err := store.connect.Database(store.database).Collection(store.collectionName).Find(ctx, where, opts) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + for cur.Next(ctx) { var data Model - err := cur.Decode(&data) - if err != nil && err != mongo.ErrNoDocuments { - return lastFileName, err + err = cur.Decode(&data) + if err != nil { + break } entry := &filer.Entry{ diff --git a/weed/filer/mysql2/mysql2_store.go b/weed/filer/mysql2/mysql2_store.go index a1f54455a..e50480150 100644 --- a/weed/filer/mysql2/mysql2_store.go +++ b/weed/filer/mysql2/mysql2_store.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "github.com/chrislusf/seaweedfs/weed/filer" @@ -82,7 +83,7 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse return fmt.Errorf("connect to %s error:%v", sqlUrl, err) } - if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { + if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") { return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) } diff --git a/weed/filer/permission.go b/weed/filer/permission.go deleted file mode 100644 index 0d8b8292b..000000000 --- a/weed/filer/permission.go +++ /dev/null @@ -1,22 +0,0 @@ -package filer - -func hasWritePermission(dir *Entry, entry *Entry) bool { - - if dir == nil { - return false - } - - if dir.Uid == entry.Uid && dir.Mode&0200 > 0 { - return true - } - - if dir.Gid == entry.Gid && dir.Mode&0020 > 0 { - return true - } - - if dir.Mode&0002 > 0 { - return true - } - - return false -} diff --git a/weed/filer/read_remote.go b/weed/filer/read_remote.go index 3406246a9..6372dac72 100644 --- a/weed/filer/read_remote.go +++ b/weed/filer/read_remote.go @@ -21,13 +21,13 @@ func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMou return remoteLocation } -func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, remoteLocationPath string)(fp util.FullPath) { +func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, remoteLocationPath string) (fp util.FullPath) { return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):]) } -func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error { - return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - _, err := client.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{ +func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error { + return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + _, err := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{ Directory: string(parent), Name: entry.Name, }) diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go index 14e8cab1e..e34034eb6 100644 --- a/weed/filer/read_write.go +++ b/weed/filer/read_write.go @@ -4,7 +4,6 @@ import ( "bytes" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/wdclient" - "math" "time" ) @@ -23,7 +22,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed return err } - return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, int64(FileSize(respLookupEntry.Entry))) } @@ -54,15 +53,14 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte Name: name, IsDirectory: false, Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(0644), - Collection: "", - Replication: "", - FileSize: uint64(len(content)), + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0644), + FileSize: uint64(len(content)), }, Content: content, }, + SkipCheckParentDirectory: true, }) } else if err == nil { entry := resp.Entry diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index 458cf88be..7d9997761 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -12,20 +12,16 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" "github.com/chrislusf/seaweedfs/weed/wdclient" - "github.com/golang/groupcache/singleflight" ) type ChunkReadAt struct { - masterClient *wdclient.MasterClient - chunkViews []*ChunkView - lookupFileId wdclient.LookupFileIdFunctionType - readerLock sync.Mutex - fileSize int64 - - fetchGroup singleflight.Group - chunkCache chunk_cache.ChunkCache - lastChunkFileId string - lastChunkData []byte + masterClient *wdclient.MasterClient + chunkViews []*ChunkView + readerLock sync.Mutex + fileSize int64 + readerCache *ReaderCache + readerPattern *ReaderPattern + lastChunkFid string } var _ = io.ReaderAt(&ChunkReadAt{}) @@ -43,7 +39,7 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp if !found { util.Retry("lookup volume "+vid, func() error { - err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ VolumeIds: []string{vid}, }) @@ -88,21 +84,22 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt { return &ChunkReadAt{ - chunkViews: chunkViews, - lookupFileId: lookupFn, - chunkCache: chunkCache, - fileSize: fileSize, + chunkViews: chunkViews, + fileSize: fileSize, + readerCache: newReaderCache(32, chunkCache, lookupFn), + readerPattern: NewReaderPattern(), } } func (c *ChunkReadAt) Close() error { - c.lastChunkData = nil - c.lastChunkFileId = "" + c.readerCache.destroy() return nil } func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) { + c.readerPattern.MonitorReadAt(offset, len(p)) + c.readerLock.Lock() defer c.readerLock.Unlock() @@ -113,19 +110,17 @@ func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) { func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { startOffset, remaining := offset, int64(len(p)) - var nextChunk *ChunkView + var nextChunks []*ChunkView for i, chunk := range c.chunkViews { if remaining <= 0 { break } if i+1 < len(c.chunkViews) { - nextChunk = c.chunkViews[i+1] - } else { - nextChunk = nil + nextChunks = c.chunkViews[i+1:] } if startOffset < chunk.LogicOffset { gap := int(chunk.LogicOffset - startOffset) - glog.V(4).Infof("zero [%d,%d)", startOffset, startOffset+int64(gap)) + glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.LogicOffset) n += int(min(int64(gap), remaining)) startOffset, remaining = chunk.LogicOffset, remaining-int64(gap) if remaining <= 0 { @@ -138,16 +133,13 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { continue } // glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.LogicOffset-chunk.Offset, chunk.LogicOffset-chunk.Offset+int64(chunk.Size)) - var buffer []byte bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset - bufferLength := chunkStop - chunkStart - buffer, err = c.readChunkSlice(chunk, nextChunk, uint64(bufferOffset), uint64(bufferLength)) + copied, err := c.readChunkSliceAt(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], chunk, nextChunks, uint64(bufferOffset)) if err != nil { glog.Errorf("fetching chunk %+v: %v\n", chunk, err) - return + return copied, err } - copied := copy(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], buffer) n += copied startOffset, remaining = startOffset+int64(copied), remaining-int64(copied) } @@ -169,77 +161,25 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { } -func (c *ChunkReadAt) readChunkSlice(chunkView *ChunkView, nextChunkViews *ChunkView, offset, length uint64) ([]byte, error) { - - chunkSlice := c.chunkCache.GetChunkSlice(chunkView.FileId, offset, length) - if len(chunkSlice) > 0 { - return chunkSlice, nil - } - chunkData, err := c.readFromWholeChunkData(chunkView, nextChunkViews) - if err != nil { - return nil, err - } - wanted := min(int64(length), int64(len(chunkData))-int64(offset)) - return chunkData[offset : int64(offset)+wanted], nil -} - -func (c *ChunkReadAt) readFromWholeChunkData(chunkView *ChunkView, nextChunkViews ...*ChunkView) (chunkData []byte, err error) { - - if c.lastChunkFileId == chunkView.FileId { - return c.lastChunkData, nil - } - - v, doErr := c.readOneWholeChunk(chunkView) +func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews []*ChunkView, offset uint64) (n int, err error) { - if doErr != nil { - return nil, doErr + if c.readerPattern.IsRandomMode() { + return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset)) } - chunkData = v.([]byte) - - c.lastChunkData = chunkData - c.lastChunkFileId = chunkView.FileId - - for _, nextChunkView := range nextChunkViews { - if c.chunkCache != nil && nextChunkView != nil { - go c.readOneWholeChunk(nextChunkView) + n, err = c.readerCache.ReadChunkAt(buffer, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset), int(chunkView.ChunkSize), chunkView.LogicOffset == 0) + if c.lastChunkFid != chunkView.FileId { + if chunkView.Offset == 0 { // start of a new chunk + if c.lastChunkFid != "" { + c.readerCache.UnCache(c.lastChunkFid) + c.readerCache.MaybeCache(nextChunkViews) + } else { + if len(nextChunkViews) >= 1 { + c.readerCache.MaybeCache(nextChunkViews[:1]) // just read the next chunk if at the very beginning + } + } } } - + c.lastChunkFid = chunkView.FileId return } - -func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, error) { - - var err error - - return c.fetchGroup.Do(chunkView.FileId, func() (interface{}, error) { - - glog.V(4).Infof("readFromWholeChunkData %s offset %d [%d,%d) size at least %d", chunkView.FileId, chunkView.Offset, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.ChunkSize) - - data := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize) - if data != nil { - glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset-chunkView.Offset, chunkView.LogicOffset-chunkView.Offset+int64(len(data))) - } else { - var err error - data, err = c.doFetchFullChunkData(chunkView) - if err != nil { - return data, err - } - c.chunkCache.SetChunk(chunkView.FileId, data) - } - return data, err - }) -} - -func (c *ChunkReadAt) doFetchFullChunkData(chunkView *ChunkView) ([]byte, error) { - - glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId) - - data, err := fetchChunk(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped) - - glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId) - - return data, err - -} diff --git a/weed/filer/reader_at_test.go b/weed/filer/reader_at_test.go index f8e4727ce..d9afb460c 100644 --- a/weed/filer/reader_at_test.go +++ b/weed/filer/reader_at_test.go @@ -21,8 +21,12 @@ func (m *mockChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) { return data } -func (m *mockChunkCache) GetChunkSlice(fileId string, offset, length uint64) []byte { - return nil +func (m *mockChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) { + x, _ := strconv.Atoi(fileId) + for i := 0; i < len(data); i++ { + data[i] = byte(x) + } + return len(data), nil } func (m *mockChunkCache) SetChunk(fileId string, data []byte) { @@ -64,11 +68,11 @@ func TestReaderAt(t *testing.T) { } readerAt := &ChunkReadAt{ - chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), - lookupFileId: nil, - readerLock: sync.Mutex{}, - fileSize: 10, - chunkCache: &mockChunkCache{}, + chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), + readerLock: sync.Mutex{}, + fileSize: 10, + readerCache: newReaderCache(3, &mockChunkCache{}, nil), + readerPattern: NewReaderPattern(), } testReadAt(t, readerAt, 0, 10, 10, io.EOF) @@ -80,7 +84,7 @@ func TestReaderAt(t *testing.T) { func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, expected int, expectedErr error) { data := make([]byte, size) - n, err := readerAt.ReadAt(data, offset) + n, err := readerAt.doReadAt(data, offset) for _, d := range data { fmt.Printf("%x", d) @@ -114,11 +118,11 @@ func TestReaderAt0(t *testing.T) { } readerAt := &ChunkReadAt{ - chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), - lookupFileId: nil, - readerLock: sync.Mutex{}, - fileSize: 10, - chunkCache: &mockChunkCache{}, + chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), + readerLock: sync.Mutex{}, + fileSize: 10, + readerCache: newReaderCache(3, &mockChunkCache{}, nil), + readerPattern: NewReaderPattern(), } testReadAt(t, readerAt, 0, 10, 10, io.EOF) @@ -142,11 +146,11 @@ func TestReaderAt1(t *testing.T) { } readerAt := &ChunkReadAt{ - chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), - lookupFileId: nil, - readerLock: sync.Mutex{}, - fileSize: 20, - chunkCache: &mockChunkCache{}, + chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), + readerLock: sync.Mutex{}, + fileSize: 20, + readerCache: newReaderCache(3, &mockChunkCache{}, nil), + readerPattern: NewReaderPattern(), } testReadAt(t, readerAt, 0, 20, 20, io.EOF) diff --git a/weed/filer/reader_cache.go b/weed/filer/reader_cache.go new file mode 100644 index 000000000..bce97cc49 --- /dev/null +++ b/weed/filer/reader_cache.go @@ -0,0 +1,192 @@ +package filer + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" + "github.com/chrislusf/seaweedfs/weed/util/mem" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "sync" + "time" +) + +type ReaderCache struct { + chunkCache chunk_cache.ChunkCache + lookupFileIdFn wdclient.LookupFileIdFunctionType + sync.Mutex + downloaders map[string]*SingleChunkCacher + limit int +} + +type SingleChunkCacher struct { + sync.RWMutex + parent *ReaderCache + chunkFileId string + data []byte + err error + cipherKey []byte + isGzipped bool + chunkSize int + shouldCache bool + wg sync.WaitGroup + completedTime time.Time +} + +func newReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache { + return &ReaderCache{ + limit: limit, + chunkCache: chunkCache, + lookupFileIdFn: lookupFileIdFn, + downloaders: make(map[string]*SingleChunkCacher), + } +} + +func (rc *ReaderCache) MaybeCache(chunkViews []*ChunkView) { + if rc.lookupFileIdFn == nil { + return + } + + rc.Lock() + defer rc.Unlock() + + for _, chunkView := range chunkViews { + if _, found := rc.downloaders[chunkView.FileId]; found { + continue + } + + if len(rc.downloaders) >= rc.limit { + // if still no slots, return + return + } + + // glog.V(4).Infof("prefetch %s offset %d", chunkView.FileId, chunkView.LogicOffset) + // cache this chunk if not yet + cacher := newSingleChunkCacher(rc, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int(chunkView.ChunkSize), false) + cacher.wg.Add(1) + go cacher.startCaching() + cacher.wg.Wait() + rc.downloaders[chunkView.FileId] = cacher + + } + + return +} + +func (rc *ReaderCache) ReadChunkAt(buffer []byte, fileId string, cipherKey []byte, isGzipped bool, offset int64, chunkSize int, shouldCache bool) (int, error) { + rc.Lock() + defer rc.Unlock() + if cacher, found := rc.downloaders[fileId]; found { + return cacher.readChunkAt(buffer, offset) + } + if shouldCache || rc.lookupFileIdFn == nil { + n, err := rc.chunkCache.ReadChunkAt(buffer, fileId, uint64(offset)) + if n > 0 { + return n, err + } + } + + if len(rc.downloaders) >= rc.limit { + oldestFid, oldestTime := "", time.Now() + for fid, downloader := range rc.downloaders { + if !downloader.completedTime.IsZero() { + if downloader.completedTime.Before(oldestTime) { + oldestFid, oldestTime = fid, downloader.completedTime + } + } + } + if oldestFid != "" { + oldDownloader := rc.downloaders[oldestFid] + delete(rc.downloaders, oldestFid) + oldDownloader.destroy() + } + } + + // glog.V(4).Infof("cache1 %s", fileId) + + cacher := newSingleChunkCacher(rc, fileId, cipherKey, isGzipped, chunkSize, shouldCache) + cacher.wg.Add(1) + go cacher.startCaching() + cacher.wg.Wait() + rc.downloaders[fileId] = cacher + + return cacher.readChunkAt(buffer, offset) +} + +func (rc *ReaderCache) UnCache(fileId string) { + rc.Lock() + defer rc.Unlock() + // glog.V(4).Infof("uncache %s", fileId) + if downloader, found := rc.downloaders[fileId]; found { + downloader.destroy() + delete(rc.downloaders, fileId) + } +} + +func (rc *ReaderCache) destroy() { + rc.Lock() + defer rc.Unlock() + + for _, downloader := range rc.downloaders { + downloader.destroy() + } + +} + +func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte, isGzipped bool, chunkSize int, shouldCache bool) *SingleChunkCacher { + t := &SingleChunkCacher{ + parent: parent, + chunkFileId: fileId, + cipherKey: cipherKey, + isGzipped: isGzipped, + chunkSize: chunkSize, + shouldCache: shouldCache, + } + return t +} + +func (s *SingleChunkCacher) startCaching() { + s.Lock() + defer s.Unlock() + + s.wg.Done() // means this has been started + + urlStrings, err := s.parent.lookupFileIdFn(s.chunkFileId) + if err != nil { + s.err = fmt.Errorf("operation LookupFileId %s failed, err: %v", s.chunkFileId, err) + return + } + + s.data = mem.Allocate(s.chunkSize) + + _, s.err = retriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0) + if s.err != nil { + mem.Free(s.data) + s.data = nil + return + } + + s.completedTime = time.Now() + if s.shouldCache { + s.parent.chunkCache.SetChunk(s.chunkFileId, s.data) + } + + return +} + +func (s *SingleChunkCacher) destroy() { + if s.data != nil { + mem.Free(s.data) + s.data = nil + } +} + +func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) { + s.RLock() + defer s.RUnlock() + + if s.err != nil { + return 0, s.err + } + + return copy(buf, s.data[offset:]), nil + +} diff --git a/weed/filer/reader_pattern.go b/weed/filer/reader_pattern.go new file mode 100644 index 000000000..b860bc577 --- /dev/null +++ b/weed/filer/reader_pattern.go @@ -0,0 +1,38 @@ +package filer + +type ReaderPattern struct { + isStreaming bool + lastReadOffset int64 +} + +// For streaming read: only cache the first chunk +// For random read: only fetch the requested range, instead of the whole chunk + +func NewReaderPattern() *ReaderPattern { + return &ReaderPattern{ + isStreaming: true, + lastReadOffset: -1, + } +} + +func (rp *ReaderPattern) MonitorReadAt(offset int64, size int) { + isStreaming := true + if rp.lastReadOffset > offset { + isStreaming = false + } + if rp.lastReadOffset == -1 { + if offset != 0 { + isStreaming = false + } + } + rp.lastReadOffset = offset + rp.isStreaming = isStreaming +} + +func (rp *ReaderPattern) IsStreamingMode() bool { + return rp.isStreaming +} + +func (rp *ReaderPattern) IsRandomMode() bool { + return !rp.isStreaming +} diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 30d11a7f4..89684647b 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -3,7 +3,7 @@ package redis import ( "context" "fmt" - "sort" + "golang.org/x/exp/slices" "strings" "time" @@ -40,7 +40,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } @@ -120,6 +120,8 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full if err != nil { return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } + // not efficient, but need to remove if it is a directory + store.Client.Del(ctx, genDirectoryListKey(string(path))) } return nil @@ -155,8 +157,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP } // sort - sort.Slice(members, func(i, j int) bool { - return strings.Compare(members[i], members[j]) < 0 + slices.SortFunc(members, func(a, b string) bool { + return strings.Compare(a, b) < 0 }) // limit diff --git a/weed/filer/redis2/redis_sentinel_store.go b/weed/filer/redis2/redis_sentinel_store.go new file mode 100644 index 000000000..802588b2b --- /dev/null +++ b/weed/filer/redis2/redis_sentinel_store.go @@ -0,0 +1,45 @@ +package redis2 + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" + "time" +) + +func init() { + filer.Stores = append(filer.Stores, &Redis2SentinelStore{}) +} + +type Redis2SentinelStore struct { + UniversalRedis2Store +} + +func (store *Redis2SentinelStore) GetName() string { + return "redis2_sentinel" +} + +func (store *Redis2SentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetStringSlice(prefix+"addresses"), + configuration.GetString(prefix+"masterName"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + ) +} + +func (store *Redis2SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) { + store.Client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: masterName, + SentinelAddrs: addresses, + Username: username, + Password: password, + DB: database, + MinRetryBackoff: time.Millisecond * 100, + MaxRetryBackoff: time.Minute * 1, + ReadTimeout: time.Second * 30, + WriteTimeout: time.Second * 5, + }) + return +} diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index aab3d1f4a..7a34092a0 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -52,7 +52,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } @@ -133,7 +133,10 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } - members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result() + members, err := store.Client.ZRangeByLex(ctx, genDirectoryListKey(string(fullpath)), &redis.ZRangeBy{ + Min: "-", + Max: "+", + }).Result() if err != nil { return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) } @@ -144,6 +147,8 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful if err != nil { return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) } + // not efficient, but need to remove if it is a directory + store.Client.Del(ctx, genDirectoryListKey(string(path))) } return nil @@ -156,14 +161,22 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { dirListKey := genDirectoryListKey(string(dirPath)) - start := int64(0) + + min := "-" if startFileName != "" { - start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result() - if !includeStartFile { - start++ + if includeStartFile { + min = "[" + startFileName + } else { + min = "(" + startFileName } } - members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result() + + members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ + Min: min, + Max: "+", + Offset: 0, + Count: limit, + }).Result() if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } diff --git a/weed/filer/redis3/ItemList.go b/weed/filer/redis3/ItemList.go new file mode 100644 index 000000000..af3b8ae5a --- /dev/null +++ b/weed/filer/redis3/ItemList.go @@ -0,0 +1,507 @@ +package redis3 + +import ( + "bytes" + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/util/skiplist" + "github.com/go-redis/redis/v8" +) + +type ItemList struct { + skipList *skiplist.SkipList + batchSize int + client redis.UniversalClient + prefix string +} + +func newItemList(client redis.UniversalClient, prefix string, store skiplist.ListStore, batchSize int) *ItemList { + return &ItemList{ + skipList: skiplist.New(store), + batchSize: batchSize, + client: client, + prefix: prefix, + } +} + +/* +Be reluctant to create new nodes. Try to fit into either previous node or next node. +Prefer to add to previous node. + +There are multiple cases after finding the name for greater or equal node + 1. found and node.Key == name + The node contains a batch with leading key the same as the name + nothing to do + 2. no such node found or node.Key > name + + if no such node found + prevNode = list.LargestNode + + // case 2.1 + if previousNode contains name + nothing to do + + // prefer to add to previous node + if prevNode != nil { + // case 2.2 + if prevNode has capacity + prevNode.add name, and save + return + // case 2.3 + split prevNode by name + } + + // case 2.4 + // merge into next node. Avoid too many nodes if adding data in reverse order. + if nextNode is not nil and nextNode has capacity + delete nextNode.Key + nextNode.Key = name + nextNode.batch.add name + insert nodeNode.Key + return + + // case 2.5 + if prevNode is nil + insert new node with key = name, value = batch{name} + return + +*/ + +func (nl *ItemList) canAddMember(node *skiplist.SkipListElementReference, name string) (alreadyContains bool, nodeSize int, err error) { + ctx := context.Background() + pipe := nl.client.TxPipeline() + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + countOperation := pipe.ZLexCount(ctx, key, "-", "+") + scoreOperationt := pipe.ZScore(ctx, key, name) + if _, err = pipe.Exec(ctx); err != nil && err != redis.Nil { + return false, 0, err + } + if err == redis.Nil { + err = nil + } + alreadyContains = scoreOperationt.Err() == nil + nodeSize = int(countOperation.Val()) + return +} + +func (nl *ItemList) WriteName(name string) error { + + lookupKey := []byte(name) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + // case 1: the name already exists as one leading key in the batch + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + return nil + } + + var prevNodeReference *skiplist.SkipListElementReference + if !found { + prevNodeReference = nl.skipList.GetLargestNodeReference() + } + + if nextNode != nil && prevNode == nil { + prevNodeReference = nextNode.Prev + } + + if prevNodeReference != nil { + alreadyContains, nodeSize, err := nl.canAddMember(prevNodeReference, name) + if err != nil { + return err + } + if alreadyContains { + // case 2.1 + return nil + } + + // case 2.2 + if nodeSize < nl.batchSize { + return nl.NodeAddMember(prevNodeReference, name) + } + + // case 2.3 + x := nl.NodeInnerPosition(prevNodeReference, name) + y := nodeSize - x + addToX := x <= y + // add to a new node + if x == 0 || y == 0 { + if err := nl.ItemAdd(lookupKey, 0, name); err != nil { + return err + } + return nil + } + if addToX { + // collect names before name, add them to X + namesToX, err := nl.NodeRangeBeforeExclusive(prevNodeReference, name) + if err != nil { + return nil + } + // delete skiplist reference to old node + if _, err := nl.skipList.DeleteByKey(prevNodeReference.Key); err != nil { + return err + } + // add namesToY and name to a new X + namesToX = append(namesToX, name) + if err := nl.ItemAdd([]byte(namesToX[0]), 0, namesToX...); err != nil { + return nil + } + // remove names less than name from current Y + if err := nl.NodeDeleteBeforeExclusive(prevNodeReference, name); err != nil { + return nil + } + + // point skip list to current Y + if err := nl.ItemAdd(lookupKey, prevNodeReference.ElementPointer); err != nil { + return nil + } + return nil + } else { + // collect names after name, add them to Y + namesToY, err := nl.NodeRangeAfterExclusive(prevNodeReference, name) + if err != nil { + return nil + } + // add namesToY and name to a new Y + namesToY = append(namesToY, name) + if err := nl.ItemAdd(lookupKey, 0, namesToY...); err != nil { + return nil + } + // remove names after name from current X + if err := nl.NodeDeleteAfterExclusive(prevNodeReference, name); err != nil { + return nil + } + return nil + } + + } + + // case 2.4 + if nextNode != nil { + nodeSize := nl.NodeSize(nextNode.Reference()) + if nodeSize < nl.batchSize { + if id, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } else { + if err := nl.ItemAdd(lookupKey, id, name); err != nil { + return err + } + } + return nil + } + } + + // case 2.5 + // now prevNode is nil + return nl.ItemAdd(lookupKey, 0, name) +} + +/* +// case 1: exists in nextNode +if nextNode != nil && nextNode.Key == name { + remove from nextNode, update nextNode + // TODO: merge with prevNode if possible? + return +} +if nextNode is nil + prevNode = list.Largestnode +if prevNode == nil and nextNode.Prev != nil + prevNode = load(nextNode.Prev) + +// case 2: does not exist +// case 2.1 +if prevNode == nil { + return +} +// case 2.2 +if prevNameBatch does not contain name { + return +} + +// case 3 +delete from prevNameBatch +if prevNameBatch + nextNode < capacityList + // case 3.1 + merge +else + // case 3.2 + update prevNode + + +*/ +func (nl *ItemList) DeleteName(name string) error { + lookupKey := []byte(name) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + + // case 1 + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } + if err := nl.NodeDeleteMember(nextNode.Reference(), name); err != nil { + return err + } + minName := nl.NodeMin(nextNode.Reference()) + if minName == "" { + return nl.NodeDelete(nextNode.Reference()) + } + return nl.ItemAdd([]byte(minName), nextNode.Id) + } + + if !found { + prevNode, err = nl.skipList.GetLargestNode() + if err != nil { + return err + } + } + + if nextNode != nil && prevNode == nil { + prevNode, err = nl.skipList.LoadElement(nextNode.Prev) + if err != nil { + return err + } + } + + // case 2 + if prevNode == nil { + // case 2.1 + return nil + } + if !nl.NodeContainsItem(prevNode.Reference(), name) { + return nil + } + + // case 3 + if err := nl.NodeDeleteMember(prevNode.Reference(), name); err != nil { + return err + } + prevSize := nl.NodeSize(prevNode.Reference()) + if prevSize == 0 { + if _, err := nl.skipList.DeleteByKey(prevNode.Key); err != nil { + return err + } + return nil + } + nextSize := nl.NodeSize(nextNode.Reference()) + if nextSize > 0 && prevSize+nextSize < nl.batchSize { + // case 3.1 merge nextNode and prevNode + if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } + nextNames, err := nl.NodeRangeBeforeExclusive(nextNode.Reference(), "") + if err != nil { + return err + } + if err := nl.NodeAddMember(prevNode.Reference(), nextNames...); err != nil { + return err + } + return nl.NodeDelete(nextNode.Reference()) + } else { + // case 3.2 update prevNode + // no action to take + return nil + } + + return nil +} + +func (nl *ItemList) ListNames(startFrom string, visitNamesFn func(name string) bool) error { + lookupKey := []byte(startFrom) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + prevNode = nil + } + if !found { + prevNode, err = nl.skipList.GetLargestNode() + if err != nil { + return err + } + } + + if prevNode != nil { + if !nl.NodeScanIncluseiveAfter(prevNode.Reference(), startFrom, visitNamesFn) { + return nil + } + } + + for nextNode != nil { + if !nl.NodeScanIncluseiveAfter(nextNode.Reference(), startFrom, visitNamesFn) { + return nil + } + nextNode, err = nl.skipList.LoadElement(nextNode.Next[0]) + if err != nil { + return err + } + } + + return nil +} + +func (nl *ItemList) RemoteAllListElement() error { + + t := nl.skipList + + nodeRef := t.StartLevels[0] + for nodeRef != nil { + node, err := t.LoadElement(nodeRef) + if err != nil { + return err + } + if node == nil { + return nil + } + if err := t.DeleteElement(node); err != nil { + return err + } + if err := nl.NodeDelete(node.Reference()); err != nil { + return err + } + nodeRef = node.Next[0] + } + return nil + +} + +func (nl *ItemList) NodeContainsItem(node *skiplist.SkipListElementReference, item string) bool { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + _, err := nl.client.ZScore(context.Background(), key, item).Result() + if err == redis.Nil { + return false + } + if err == nil { + return true + } + return false +} + +func (nl *ItemList) NodeSize(node *skiplist.SkipListElementReference) int { + if node == nil { + return 0 + } + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + return int(nl.client.ZLexCount(context.Background(), key, "-", "+").Val()) +} + +func (nl *ItemList) NodeAddMember(node *skiplist.SkipListElementReference, names ...string) error { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + var members []*redis.Z + for _, name := range names { + members = append(members, &redis.Z{ + Score: 0, + Member: name, + }) + } + return nl.client.ZAddNX(context.Background(), key, members...).Err() +} +func (nl *ItemList) NodeDeleteMember(node *skiplist.SkipListElementReference, name string) error { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + return nl.client.ZRem(context.Background(), key, name).Err() +} + +func (nl *ItemList) NodeDelete(node *skiplist.SkipListElementReference) error { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + return nl.client.Del(context.Background(), key).Err() +} + +func (nl *ItemList) NodeInnerPosition(node *skiplist.SkipListElementReference, name string) int { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + return int(nl.client.ZLexCount(context.Background(), key, "-", "("+name).Val()) +} + +func (nl *ItemList) NodeMin(node *skiplist.SkipListElementReference) string { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + slice := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{ + Min: "-", + Max: "+", + Offset: 0, + Count: 1, + }).Val() + if len(slice) > 0 { + s := slice[0] + return s + } + return "" +} + +func (nl *ItemList) NodeScanIncluseiveAfter(node *skiplist.SkipListElementReference, startFrom string, visitNamesFn func(name string) bool) bool { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + if startFrom == "" { + startFrom = "-" + } else { + startFrom = "[" + startFrom + } + names := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{ + Min: startFrom, + Max: "+", + }).Val() + for _, n := range names { + if !visitNamesFn(n) { + return false + } + } + return true +} + +func (nl *ItemList) NodeRangeBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) ([]string, error) { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + if stopAt == "" { + stopAt = "+" + } else { + stopAt = "(" + stopAt + } + return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{ + Min: "-", + Max: stopAt, + }).Result() +} +func (nl *ItemList) NodeRangeAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) ([]string, error) { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + if startFrom == "" { + startFrom = "-" + } else { + startFrom = "(" + startFrom + } + return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{ + Min: startFrom, + Max: "+", + }).Result() +} + +func (nl *ItemList) NodeDeleteBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) error { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + if stopAt == "" { + stopAt = "+" + } else { + stopAt = "(" + stopAt + } + return nl.client.ZRemRangeByLex(context.Background(), key, "-", stopAt).Err() +} +func (nl *ItemList) NodeDeleteAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) error { + key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) + if startFrom == "" { + startFrom = "-" + } else { + startFrom = "(" + startFrom + } + return nl.client.ZRemRangeByLex(context.Background(), key, startFrom, "+").Err() +} + +func (nl *ItemList) ItemAdd(lookupKey []byte, idIfKnown int64, names ...string) error { + if id, err := nl.skipList.InsertByKey(lookupKey, idIfKnown, nil); err != nil { + return err + } else { + if len(names) > 0 { + return nl.NodeAddMember(&skiplist.SkipListElementReference{ + ElementPointer: id, + Key: lookupKey, + }, names...) + } + } + return nil +} diff --git a/weed/filer/redis3/item_list_serde.go b/weed/filer/redis3/item_list_serde.go new file mode 100644 index 000000000..d0310ce40 --- /dev/null +++ b/weed/filer/redis3/item_list_serde.go @@ -0,0 +1,75 @@ +package redis3 + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util/skiplist" + "github.com/go-redis/redis/v8" + "github.com/golang/protobuf/proto" +) + +func LoadItemList(data []byte, prefix string, client redis.UniversalClient, store skiplist.ListStore, batchSize int) *ItemList { + + nl := &ItemList{ + skipList: skiplist.New(store), + batchSize: batchSize, + client: client, + prefix: prefix, + } + + if len(data) == 0 { + return nl + } + + message := &skiplist.SkipListProto{} + if err := proto.Unmarshal(data, message); err != nil { + glog.Errorf("loading skiplist: %v", err) + } + nl.skipList.MaxNewLevel = int(message.MaxNewLevel) + nl.skipList.MaxLevel = int(message.MaxLevel) + for i, ref := range message.StartLevels { + nl.skipList.StartLevels[i] = &skiplist.SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + } + } + for i, ref := range message.EndLevels { + nl.skipList.EndLevels[i] = &skiplist.SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + } + } + return nl +} + +func (nl *ItemList) HasChanges() bool { + return nl.skipList.HasChanges +} + +func (nl *ItemList) ToBytes() []byte { + message := &skiplist.SkipListProto{} + message.MaxNewLevel = int32(nl.skipList.MaxNewLevel) + message.MaxLevel = int32(nl.skipList.MaxLevel) + for _, ref := range nl.skipList.StartLevels { + if ref == nil { + break + } + message.StartLevels = append(message.StartLevels, &skiplist.SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + }) + } + for _, ref := range nl.skipList.EndLevels { + if ref == nil { + break + } + message.EndLevels = append(message.EndLevels, &skiplist.SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + }) + } + data, err := proto.Marshal(message) + if err != nil { + glog.Errorf("marshal skiplist: %v", err) + } + return data +} diff --git a/weed/filer/redis3/kv_directory_children.go b/weed/filer/redis3/kv_directory_children.go new file mode 100644 index 000000000..d92dddfe6 --- /dev/null +++ b/weed/filer/redis3/kv_directory_children.go @@ -0,0 +1,138 @@ +package redis3 + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/go-redis/redis/v8" +) + +const maxNameBatchSizeLimit = 1000000 + +func insertChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error { + + // lock and unlock + mutex := redisStore.redsync.NewMutex(key + "lock") + if err := mutex.Lock(); err != nil { + return fmt.Errorf("lock %s: %v", key, err) + } + defer func() { + mutex.Unlock() + }() + + client := redisStore.Client + data, err := client.Get(ctx, key).Result() + if err != nil { + if err != redis.Nil { + return fmt.Errorf("read %s: %v", key, err) + } + } + store := newSkipListElementStore(key, client) + nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit) + + if err := nameList.WriteName(name); err != nil { + glog.Errorf("add %s %s: %v", key, name, err) + return err + } + + if !nameList.HasChanges() { + return nil + } + + if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil { + return err + } + + return nil +} + +func removeChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error { + + // lock and unlock + mutex := redisStore.redsync.NewMutex(key + "lock") + if err := mutex.Lock(); err != nil { + return fmt.Errorf("lock %s: %v", key, err) + } + defer mutex.Unlock() + + client := redisStore.Client + data, err := client.Get(ctx, key).Result() + if err != nil { + if err != redis.Nil { + return fmt.Errorf("read %s: %v", key, err) + } + } + store := newSkipListElementStore(key, client) + nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit) + + if err := nameList.DeleteName(name); err != nil { + return err + } + if !nameList.HasChanges() { + return nil + } + + if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil { + return err + } + + return nil +} + +func removeChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, onDeleteFn func(name string) error) error { + + // lock and unlock + mutex := redisStore.redsync.NewMutex(key + "lock") + if err := mutex.Lock(); err != nil { + return fmt.Errorf("lock %s: %v", key, err) + } + defer mutex.Unlock() + + client := redisStore.Client + data, err := client.Get(ctx, key).Result() + if err != nil { + if err != redis.Nil { + return fmt.Errorf("read %s: %v", key, err) + } + } + store := newSkipListElementStore(key, client) + nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit) + + if err = nameList.ListNames("", func(name string) bool { + if err := onDeleteFn(name); err != nil { + glog.Errorf("delete %s child %s: %v", key, name, err) + return false + } + return true + }); err != nil { + return err + } + + if err = nameList.RemoteAllListElement(); err != nil { + return err + } + + return nil + +} + +func listChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, startFileName string, eachFn func(name string) bool) error { + client := redisStore.Client + data, err := client.Get(ctx, key).Result() + if err != nil { + if err != redis.Nil { + return fmt.Errorf("read %s: %v", key, err) + } + } + store := newSkipListElementStore(key, client) + nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit) + + if err = nameList.ListNames(startFileName, func(name string) bool { + return eachFn(name) + }); err != nil { + return err + } + + return nil + +} diff --git a/weed/filer/redis3/kv_directory_children_test.go b/weed/filer/redis3/kv_directory_children_test.go new file mode 100644 index 000000000..9d7acacf1 --- /dev/null +++ b/weed/filer/redis3/kv_directory_children_test.go @@ -0,0 +1,210 @@ +package redis3 + +import ( + "context" + "fmt" + "github.com/go-redis/redis/v8" + "github.com/stvp/tempredis" + "strconv" + "testing" + "time" +) + +var names = []string{ + "cassandra.in.sh", + "cassandra", + "debug-cql.bat", + "nodetool", + "nodetool.bat", + "source-conf.ps1", + "sstableloader", + "sstableloader.bat", + "sstablescrub", + "sstablescrub.bat", + "sstableupgrade", + "sstableupgrade.bat", + "sstableutil", + "sstableutil.bat", + "sstableverify", + "sstableverify.bat", + "stop-server", + "stop-server.bat", + "stop-server.ps1", + "cassandra.in.bat", + "cqlsh.py", + "cqlsh", + "cassandra.ps1", + "cqlsh.bat", + "debug-cql", + "cassandra.bat", +} + +func yTestNameList(t *testing.T) { + server, err := tempredis.Start(tempredis.Config{}) + if err != nil { + panic(err) + } + defer server.Term() + + client := redis.NewClient(&redis.Options{ + Network: "unix", + Addr: server.Socket(), + }) + + store := newSkipListElementStore("/yyy/bin", client) + var data []byte + for _, name := range names { + nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit) + nameList.WriteName(name) + + nameList.ListNames("", func(name string) bool { + println(name) + return true + }) + + if nameList.HasChanges() { + data = nameList.ToBytes() + } + println() + } + + nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit) + nameList.ListNames("", func(name string) bool { + println(name) + return true + }) + +} + +func yBenchmarkNameList(b *testing.B) { + + server, err := tempredis.Start(tempredis.Config{}) + if err != nil { + panic(err) + } + defer server.Term() + + client := redis.NewClient(&redis.Options{ + Network: "unix", + Addr: server.Socket(), + }) + + store := newSkipListElementStore("/yyy/bin", client) + var data []byte + for i := 0; i < b.N; i++ { + nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit) + + nameList.WriteName(strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx") + + if nameList.HasChanges() { + data = nameList.ToBytes() + } + } +} + +func BenchmarkRedis(b *testing.B) { + + server, err := tempredis.Start(tempredis.Config{}) + if err != nil { + panic(err) + } + defer server.Term() + + client := redis.NewClient(&redis.Options{ + Network: "unix", + Addr: server.Socket(), + }) + + for i := 0; i < b.N; i++ { + client.ZAddNX(context.Background(), "/yyy/bin", &redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"}) + } +} + +func xTestNameListAdd(t *testing.T) { + + server, err := tempredis.Start(tempredis.Config{}) + if err != nil { + panic(err) + } + defer server.Term() + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + client.FlushAll(context.Background()) + + N := 364800 + + ts0 := time.Now() + store := newSkipListElementStore("/y", client) + var data []byte + nameList := LoadItemList(data, "/y", client, store, 100000) + for i := 0; i < N; i++ { + nameList.WriteName(fmt.Sprintf("%8d", i)) + } + + ts1 := time.Now() + + for i := 0; i < N; i++ { + client.ZAddNX(context.Background(), "/x", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) + } + ts2 := time.Now() + + fmt.Printf("%v %v", ts1.Sub(ts0), ts2.Sub(ts1)) + + /* + keys := client.Keys(context.Background(), "/*m").Val() + for _, k := range keys { + println("key", k) + for i, v := range client.ZRangeByLex(context.Background(), k, &redis.ZRangeBy{ + Min: "-", + Max: "+", + }).Val() { + println(" ", i, v) + } + } + */ +} + +func xBenchmarkNameList(b *testing.B) { + + server, err := tempredis.Start(tempredis.Config{}) + if err != nil { + panic(err) + } + defer server.Term() + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + store := newSkipListElementStore("/yyy/bin", client) + var data []byte + for i := 0; i < b.N; i++ { + nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit) + + nameList.WriteName(fmt.Sprintf("name %8d", i)) + + if nameList.HasChanges() { + data = nameList.ToBytes() + } + } +} + +func xBenchmarkRedis(b *testing.B) { + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + for i := 0; i < b.N; i++ { + client.ZAddNX(context.Background(), "/xxx/bin", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) + } +} diff --git a/weed/filer/redis3/redis_cluster_store.go b/weed/filer/redis3/redis_cluster_store.go new file mode 100644 index 000000000..73fc0dd20 --- /dev/null +++ b/weed/filer/redis3/redis_cluster_store.go @@ -0,0 +1,45 @@ +package redis3 + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" + "github.com/go-redsync/redsync/v4" + "github.com/go-redsync/redsync/v4/redis/goredis/v8" +) + +func init() { + filer.Stores = append(filer.Stores, &RedisCluster3Store{}) +} + +type RedisCluster3Store struct { + UniversalRedis3Store +} + +func (store *RedisCluster3Store) GetName() string { + return "redis_cluster3" +} + +func (store *RedisCluster3Store) Initialize(configuration util.Configuration, prefix string) (err error) { + + configuration.SetDefault(prefix+"useReadOnly", false) + configuration.SetDefault(prefix+"routeByLatency", false) + + return store.initialize( + configuration.GetStringSlice(prefix+"addresses"), + configuration.GetString(prefix+"password"), + configuration.GetBool(prefix+"useReadOnly"), + configuration.GetBool(prefix+"routeByLatency"), + ) +} + +func (store *RedisCluster3Store) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) { + store.Client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: addresses, + Password: password, + ReadOnly: readOnly, + RouteByLatency: routeByLatency, + }) + store.redsync = redsync.New(goredis.NewPool(store.Client)) + return +} diff --git a/weed/filer/redis3/redis_sentinel_store.go b/weed/filer/redis3/redis_sentinel_store.go new file mode 100644 index 000000000..a87302167 --- /dev/null +++ b/weed/filer/redis3/redis_sentinel_store.go @@ -0,0 +1,49 @@ +package redis3 + +import ( + "time" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" + "github.com/go-redsync/redsync/v4" + "github.com/go-redsync/redsync/v4/redis/goredis/v8" +) + +func init() { + filer.Stores = append(filer.Stores, &Redis3SentinelStore{}) +} + +type Redis3SentinelStore struct { + UniversalRedis3Store +} + +func (store *Redis3SentinelStore) GetName() string { + return "redis3_sentinel" +} + +func (store *Redis3SentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetStringSlice(prefix+"addresses"), + configuration.GetString(prefix+"masterName"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + ) +} + +func (store *Redis3SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) { + store.Client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: masterName, + SentinelAddrs: addresses, + Username: username, + Password: password, + DB: database, + MinRetryBackoff: time.Millisecond * 100, + MaxRetryBackoff: time.Minute * 1, + ReadTimeout: time.Second * 30, + WriteTimeout: time.Second * 5, + }) + store.redsync = redsync.New(goredis.NewPool(store.Client)) + return +} diff --git a/weed/filer/redis3/redis_store.go b/weed/filer/redis3/redis_store.go new file mode 100644 index 000000000..2eec3d37a --- /dev/null +++ b/weed/filer/redis3/redis_store.go @@ -0,0 +1,39 @@ +package redis3 + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" + "github.com/go-redsync/redsync/v4" + "github.com/go-redsync/redsync/v4/redis/goredis/v8" +) + +func init() { + filer.Stores = append(filer.Stores, &Redis3Store{}) +} + +type Redis3Store struct { + UniversalRedis3Store +} + +func (store *Redis3Store) GetName() string { + return "redis3" +} + +func (store *Redis3Store) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"address"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + ) +} + +func (store *Redis3Store) initialize(hostPort string, password string, database int) (err error) { + store.Client = redis.NewClient(&redis.Options{ + Addr: hostPort, + Password: password, + DB: database, + }) + store.redsync = redsync.New(goredis.NewPool(store.Client)) + return +} diff --git a/weed/filer/redis3/skiplist_element_store.go b/weed/filer/redis3/skiplist_element_store.go new file mode 100644 index 000000000..8c101d006 --- /dev/null +++ b/weed/filer/redis3/skiplist_element_store.go @@ -0,0 +1,62 @@ +package redis3 + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util/skiplist" + "github.com/go-redis/redis/v8" + "github.com/golang/protobuf/proto" +) + +type SkipListElementStore struct { + Prefix string + client redis.UniversalClient +} + +var _ = skiplist.ListStore(&SkipListElementStore{}) + +func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore { + return &SkipListElementStore{ + Prefix: prefix, + client: client, + } +} + +func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error { + key := fmt.Sprintf("%s%d", m.Prefix, id) + data, err := proto.Marshal(element) + if err != nil { + glog.Errorf("marshal %s: %v", key, err) + } + return m.client.Set(context.Background(), key, data, 0).Err() +} + +func (m *SkipListElementStore) DeleteElement(id int64) error { + key := fmt.Sprintf("%s%d", m.Prefix, id) + return m.client.Del(context.Background(), key).Err() +} + +func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) { + key := fmt.Sprintf("%s%d", m.Prefix, id) + data, err := m.client.Get(context.Background(), key).Result() + if err != nil { + if err == redis.Nil { + return nil, nil + } + return nil, err + } + t := &skiplist.SkipListElement{} + err = proto.Unmarshal([]byte(data), t) + if err == nil { + for i := 0; i < len(t.Next); i++ { + if t.Next[i].IsNil() { + t.Next[i] = nil + } + } + if t.Prev.IsNil() { + t.Prev = nil + } + } + return t, err +} diff --git a/weed/filer/redis3/universal_redis_store.go b/weed/filer/redis3/universal_redis_store.go new file mode 100644 index 000000000..10a87e2a4 --- /dev/null +++ b/weed/filer/redis3/universal_redis_store.go @@ -0,0 +1,179 @@ +package redis3 + +import ( + "context" + "fmt" + "github.com/go-redsync/redsync/v4" + "time" + + "github.com/go-redis/redis/v8" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +const ( + DIR_LIST_MARKER = "\x00" +) + +type UniversalRedis3Store struct { + Client redis.UniversalClient + redsync *redsync.Redsync +} + +func (store *UniversalRedis3Store) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *UniversalRedis3Store) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + value = util.MaybeGzipData(value) + } + + if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + dir, name := entry.FullPath.DirAndName() + + if name != "" { + if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil { + return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) + } + } + + return nil +} + +func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + + return store.InsertEntry(ctx, entry) +} + +func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + + data, err := store.Client.Get(ctx, string(fullpath)).Result() + if err == redis.Nil { + return nil, filer_pb.ErrNotFound + } + + if err != nil { + return nil, fmt.Errorf("get %s : %v", fullpath, err) + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *UniversalRedis3Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + + _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result() + if err != nil { + return fmt.Errorf("delete dir list %s : %v", fullpath, err) + } + + _, err = store.Client.Del(ctx, string(fullpath)).Result() + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + dir, name := fullpath.DirAndName() + + if name != "" { + if err = removeChild(ctx, store, genDirectoryListKey(dir), name); err != nil { + return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err) + } + } + + return nil +} + +func (store *UniversalRedis3Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + + return removeChildren(ctx, store, genDirectoryListKey(string(fullpath)), func(name string) error { + path := util.NewFullPath(string(fullpath), name) + _, err = store.Client.Del(ctx, string(path)).Result() + if err != nil { + return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) + } + // not efficient, but need to remove if it is a directory + store.Client.Del(ctx, genDirectoryListKey(string(path))) + return nil + }) + +} + +func (store *UniversalRedis3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed +} + +func (store *UniversalRedis3Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + + dirListKey := genDirectoryListKey(string(dirPath)) + counter := int64(0) + + err = listChildren(ctx, store, dirListKey, startFileName, func(fileName string) bool { + if startFileName != "" { + if !includeStartFile && startFileName == fileName { + return true + } + } + + path := util.NewFullPath(string(dirPath), fileName) + entry, err := store.FindEntry(ctx, path) + lastFileName = fileName + if err != nil { + glog.V(0).Infof("list %s : %v", path, err) + if err == filer_pb.ErrNotFound { + return true + } + } else { + if entry.TtlSec > 0 { + if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + store.Client.Del(ctx, string(path)).Result() + store.Client.ZRem(ctx, dirListKey, fileName).Result() + return true + } + } + counter++ + if !eachEntryFunc(entry) { + return false + } + if counter >= limit { + return false + } + } + return true + }) + + return lastFileName, err +} + +func genDirectoryListKey(dir string) (dirList string) { + return dir + DIR_LIST_MARKER +} + +func (store *UniversalRedis3Store) Shutdown() { + store.Client.Close() +} diff --git a/weed/filer/redis3/universal_redis_store_kv.go b/weed/filer/redis3/universal_redis_store_kv.go new file mode 100644 index 000000000..a9c440a37 --- /dev/null +++ b/weed/filer/redis3/universal_redis_store_kv.go @@ -0,0 +1,42 @@ +package redis3 + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/go-redis/redis/v8" +) + +func (store *UniversalRedis3Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.Client.Set(ctx, string(key), value, 0).Result() + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *UniversalRedis3Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + data, err := store.Client.Get(ctx, string(key)).Result() + + if err == redis.Nil { + return nil, filer.ErrKvNotFound + } + + return []byte(data), err +} + +func (store *UniversalRedis3Store) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.Client.Del(ctx, string(key)).Result() + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/redis_lua/redis_cluster_store.go b/weed/filer/redis_lua/redis_cluster_store.go new file mode 100644 index 000000000..b68d1092c --- /dev/null +++ b/weed/filer/redis_lua/redis_cluster_store.go @@ -0,0 +1,44 @@ +package redis_lua + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" +) + +func init() { + filer.Stores = append(filer.Stores, &RedisLuaClusterStore{}) +} + +type RedisLuaClusterStore struct { + UniversalRedisLuaStore +} + +func (store *RedisLuaClusterStore) GetName() string { + return "redis_lua_cluster" +} + +func (store *RedisLuaClusterStore) Initialize(configuration util.Configuration, prefix string) (err error) { + + configuration.SetDefault(prefix+"useReadOnly", false) + configuration.SetDefault(prefix+"routeByLatency", false) + + return store.initialize( + configuration.GetStringSlice(prefix+"addresses"), + configuration.GetString(prefix+"password"), + configuration.GetBool(prefix+"useReadOnly"), + configuration.GetBool(prefix+"routeByLatency"), + configuration.GetStringSlice(prefix+"superLargeDirectories"), + ) +} + +func (store *RedisLuaClusterStore) initialize(addresses []string, password string, readOnly, routeByLatency bool, superLargeDirectories []string) (err error) { + store.Client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: addresses, + Password: password, + ReadOnly: readOnly, + RouteByLatency: routeByLatency, + }) + store.loadSuperLargeDirectories(superLargeDirectories) + return +} diff --git a/weed/filer/redis_lua/redis_sentinel_store.go b/weed/filer/redis_lua/redis_sentinel_store.go new file mode 100644 index 000000000..5530c098e --- /dev/null +++ b/weed/filer/redis_lua/redis_sentinel_store.go @@ -0,0 +1,45 @@ +package redis_lua + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" + "time" +) + +func init() { + filer.Stores = append(filer.Stores, &RedisLuaSentinelStore{}) +} + +type RedisLuaSentinelStore struct { + UniversalRedisLuaStore +} + +func (store *RedisLuaSentinelStore) GetName() string { + return "redis_lua_sentinel" +} + +func (store *RedisLuaSentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetStringSlice(prefix+"addresses"), + configuration.GetString(prefix+"masterName"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + ) +} + +func (store *RedisLuaSentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) { + store.Client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: masterName, + SentinelAddrs: addresses, + Username: username, + Password: password, + DB: database, + MinRetryBackoff: time.Millisecond * 100, + MaxRetryBackoff: time.Minute * 1, + ReadTimeout: time.Second * 30, + WriteTimeout: time.Second * 5, + }) + return +} diff --git a/weed/filer/redis_lua/redis_store.go b/weed/filer/redis_lua/redis_store.go new file mode 100644 index 000000000..a7d11c73c --- /dev/null +++ b/weed/filer/redis_lua/redis_store.go @@ -0,0 +1,38 @@ +package redis_lua + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/go-redis/redis/v8" +) + +func init() { + filer.Stores = append(filer.Stores, &RedisLuaStore{}) +} + +type RedisLuaStore struct { + UniversalRedisLuaStore +} + +func (store *RedisLuaStore) GetName() string { + return "redis_lua" +} + +func (store *RedisLuaStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"address"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + configuration.GetStringSlice(prefix+"superLargeDirectories"), + ) +} + +func (store *RedisLuaStore) initialize(hostPort string, password string, database int, superLargeDirectories []string) (err error) { + store.Client = redis.NewClient(&redis.Options{ + Addr: hostPort, + Password: password, + DB: database, + }) + store.loadSuperLargeDirectories(superLargeDirectories) + return +} diff --git a/weed/filer/redis_lua/stored_procedure/delete_entry.lua b/weed/filer/redis_lua/stored_procedure/delete_entry.lua new file mode 100644 index 000000000..445337c77 --- /dev/null +++ b/weed/filer/redis_lua/stored_procedure/delete_entry.lua @@ -0,0 +1,19 @@ +-- KEYS[1]: full path of entry +local fullpath = KEYS[1] +-- KEYS[2]: full path of entry +local fullpath_list_key = KEYS[2] +-- KEYS[3]: dir of the entry +local dir_list_key = KEYS[3] + +-- ARGV[1]: isSuperLargeDirectory +local isSuperLargeDirectory = ARGV[1] == "1" +-- ARGV[2]: name of the entry +local name = ARGV[2] + +redis.call("DEL", fullpath, fullpath_list_key) + +if not isSuperLargeDirectory and name ~= "" then + redis.call("ZREM", dir_list_key, name) +end + +return 0 \ No newline at end of file diff --git a/weed/filer/redis_lua/stored_procedure/delete_folder_children.lua b/weed/filer/redis_lua/stored_procedure/delete_folder_children.lua new file mode 100644 index 000000000..77e4839f9 --- /dev/null +++ b/weed/filer/redis_lua/stored_procedure/delete_folder_children.lua @@ -0,0 +1,15 @@ +-- KEYS[1]: full path of entry +local fullpath = KEYS[1] + +if fullpath ~= "" and string.sub(fullpath, -1) == "/" then + fullpath = string.sub(fullpath, 0, -2) +end + +local files = redis.call("ZRANGE", fullpath .. "\0", "0", "-1") + +for _, name in ipairs(files) do + local file_path = fullpath .. "/" .. name + redis.call("DEL", file_path, file_path .. "\0") +end + +return 0 \ No newline at end of file diff --git a/weed/filer/redis_lua/stored_procedure/init.go b/weed/filer/redis_lua/stored_procedure/init.go new file mode 100644 index 000000000..1412ceba2 --- /dev/null +++ b/weed/filer/redis_lua/stored_procedure/init.go @@ -0,0 +1,24 @@ +package stored_procedure + +import ( + _ "embed" + "github.com/go-redis/redis/v8" +) + +func init() { + InsertEntryScript = redis.NewScript(insertEntry) + DeleteEntryScript = redis.NewScript(deleteEntry) + DeleteFolderChildrenScript = redis.NewScript(deleteFolderChildren) +} + +//go:embed insert_entry.lua +var insertEntry string +var InsertEntryScript *redis.Script + +//go:embed delete_entry.lua +var deleteEntry string +var DeleteEntryScript *redis.Script + +//go:embed delete_folder_children.lua +var deleteFolderChildren string +var DeleteFolderChildrenScript *redis.Script diff --git a/weed/filer/redis_lua/stored_procedure/insert_entry.lua b/weed/filer/redis_lua/stored_procedure/insert_entry.lua new file mode 100644 index 000000000..8deef3446 --- /dev/null +++ b/weed/filer/redis_lua/stored_procedure/insert_entry.lua @@ -0,0 +1,27 @@ +-- KEYS[1]: full path of entry +local full_path = KEYS[1] +-- KEYS[2]: dir of the entry +local dir_list_key = KEYS[2] + +-- ARGV[1]: content of the entry +local entry = ARGV[1] +-- ARGV[2]: TTL of the entry +local ttlSec = tonumber(ARGV[2]) +-- ARGV[3]: isSuperLargeDirectory +local isSuperLargeDirectory = ARGV[3] == "1" +-- ARGV[4]: zscore of the entry in zset +local zscore = tonumber(ARGV[4]) +-- ARGV[5]: name of the entry +local name = ARGV[5] + +if ttlSec > 0 then + redis.call("SET", full_path, entry, "EX", ttlSec) +else + redis.call("SET", full_path, entry) +end + +if not isSuperLargeDirectory and name ~= "" then + redis.call("ZADD", dir_list_key, "NX", zscore, name) +end + +return 0 \ No newline at end of file diff --git a/weed/filer/redis_lua/universal_redis_store.go b/weed/filer/redis_lua/universal_redis_store.go new file mode 100644 index 000000000..0ab0f2f24 --- /dev/null +++ b/weed/filer/redis_lua/universal_redis_store.go @@ -0,0 +1,191 @@ +package redis_lua + +import ( + "context" + "fmt" + "time" + + "github.com/go-redis/redis/v8" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/redis_lua/stored_procedure" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +const ( + DIR_LIST_MARKER = "\x00" +) + +type UniversalRedisLuaStore struct { + Client redis.UniversalClient + superLargeDirectoryHash map[string]bool +} + +func (store *UniversalRedisLuaStore) isSuperLargeDirectory(dir string) (isSuperLargeDirectory bool) { + _, isSuperLargeDirectory = store.superLargeDirectoryHash[dir] + return +} + +func (store *UniversalRedisLuaStore) loadSuperLargeDirectories(superLargeDirectories []string) { + // set directory hash + store.superLargeDirectoryHash = make(map[string]bool) + for _, dir := range superLargeDirectories { + store.superLargeDirectoryHash[dir] = true + } +} + +func (store *UniversalRedisLuaStore) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *UniversalRedisLuaStore) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + value = util.MaybeGzipData(value) + } + + dir, name := entry.FullPath.DirAndName() + + err = stored_procedure.InsertEntryScript.Run(ctx, store.Client, + []string{string(entry.FullPath), genDirectoryListKey(dir)}, + value, entry.TtlSec, + store.isSuperLargeDirectory(dir), 0, name).Err() + + if err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + + return store.InsertEntry(ctx, entry) +} + +func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + + data, err := store.Client.Get(ctx, string(fullpath)).Result() + if err == redis.Nil { + return nil, filer_pb.ErrNotFound + } + + if err != nil { + return nil, fmt.Errorf("get %s : %v", fullpath, err) + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *UniversalRedisLuaStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + + dir, name := fullpath.DirAndName() + + err = stored_procedure.DeleteEntryScript.Run(ctx, store.Client, + []string{string(fullpath), genDirectoryListKey(string(fullpath)), genDirectoryListKey(dir)}, + store.isSuperLargeDirectory(dir), name).Err() + + if err != nil { + return fmt.Errorf("DeleteEntry %s : %v", fullpath, err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + + if store.isSuperLargeDirectory(string(fullpath)) { + return nil + } + + err = stored_procedure.DeleteFolderChildrenScript.Run(ctx, store.Client, + []string{string(fullpath)}).Err() + + if err != nil { + return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed +} + +func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + + dirListKey := genDirectoryListKey(string(dirPath)) + + min := "-" + if startFileName != "" { + if includeStartFile { + min = "[" + startFileName + } else { + min = "(" + startFileName + } + } + + members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ + Min: min, + Max: "+", + Offset: 0, + Count: limit, + }).Result() + if err != nil { + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) + } + + // fetch entry meta + for _, fileName := range members { + path := util.NewFullPath(string(dirPath), fileName) + entry, err := store.FindEntry(ctx, path) + lastFileName = fileName + if err != nil { + glog.V(0).Infof("list %s : %v", path, err) + if err == filer_pb.ErrNotFound { + continue + } + } else { + if entry.TtlSec > 0 { + if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + store.DeleteEntry(ctx, path) + continue + } + } + if !eachEntryFunc(entry) { + break + } + } + } + + return lastFileName, err +} + +func genDirectoryListKey(dir string) (dirList string) { + return dir + DIR_LIST_MARKER +} + +func (store *UniversalRedisLuaStore) Shutdown() { + store.Client.Close() +} diff --git a/weed/filer/redis_lua/universal_redis_store_kv.go b/weed/filer/redis_lua/universal_redis_store_kv.go new file mode 100644 index 000000000..3df980b66 --- /dev/null +++ b/weed/filer/redis_lua/universal_redis_store_kv.go @@ -0,0 +1,42 @@ +package redis_lua + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/go-redis/redis/v8" +) + +func (store *UniversalRedisLuaStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.Client.Set(ctx, string(key), value, 0).Result() + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + data, err := store.Client.Get(ctx, string(key)).Result() + + if err == redis.Nil { + return nil, filer.ErrKvNotFound + } + + return []byte(data), err +} + +func (store *UniversalRedisLuaStore) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.Client.Del(ctx, string(key)).Result() + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/remote_mapping.go b/weed/filer/remote_mapping.go new file mode 100644 index 000000000..b0534e2ca --- /dev/null +++ b/weed/filer/remote_mapping.go @@ -0,0 +1,121 @@ +package filer + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" +) + +func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error) { + var oldContent []byte + if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE) + return readErr + }); readErr != nil { + return nil, readErr + } + + mappings, readErr = UnmarshalRemoteStorageMappings(oldContent) + if readErr != nil { + return nil, fmt.Errorf("unmarshal mappings: %v", readErr) + } + + return +} + +func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStorageLocation *remote_pb.RemoteStorageLocation) (err error) { + + // read current mapping + var oldContent, newContent []byte + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE) + return err + }) + if err != nil { + if err != filer_pb.ErrNotFound { + return fmt.Errorf("read existing mapping: %v", err) + } + } + + // add new mapping + newContent, err = addRemoteStorageMapping(oldContent, dir, remoteStorageLocation) + if err != nil { + return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err) + } + + // save back + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent) + }) + if err != nil { + return fmt.Errorf("save mapping: %v", err) + } + + return nil +} + +func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error) { + + // read current mapping + var oldContent, newContent []byte + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + oldContent, err = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE) + return err + }) + if err != nil { + if err != filer_pb.ErrNotFound { + return fmt.Errorf("read existing mapping: %v", err) + } + } + + // add new mapping + newContent, err = removeRemoteStorageMapping(oldContent, dir) + if err != nil { + return fmt.Errorf("delete mount %s: %v", dir, err) + } + + // save back + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent) + }) + if err != nil { + return fmt.Errorf("save mapping: %v", err) + } + + return nil +} + +func addRemoteStorageMapping(oldContent []byte, dir string, storageLocation *remote_pb.RemoteStorageLocation) (newContent []byte, err error) { + mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent) + if unmarshalErr != nil { + // skip + } + + // set the new mapping + mappings.Mappings[dir] = storageLocation + + if newContent, err = proto.Marshal(mappings); err != nil { + return oldContent, fmt.Errorf("marshal mappings: %v", err) + } + + return +} + +func removeRemoteStorageMapping(oldContent []byte, dir string) (newContent []byte, err error) { + mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent) + if unmarshalErr != nil { + return nil, unmarshalErr + } + + // set the new mapping + delete(mappings.Mappings, dir) + + if newContent, err = proto.Marshal(mappings); err != nil { + return oldContent, fmt.Errorf("marshal mappings: %v", err) + } + + return +} diff --git a/weed/filer/filer_remote_storage.go b/weed/filer/remote_storage.go similarity index 75% rename from weed/filer/filer_remote_storage.go rename to weed/filer/remote_storage.go index 65704e652..5362ba738 100644 --- a/weed/filer/filer_remote_storage.go +++ b/weed/filer/remote_storage.go @@ -131,58 +131,9 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.Remo return } -func AddRemoteStorageMapping(oldContent []byte, dir string, storageLocation *remote_pb.RemoteStorageLocation) (newContent []byte, err error) { - mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent) - if unmarshalErr != nil { - // skip - } - - // set the new mapping - mappings.Mappings[dir] = storageLocation - - if newContent, err = proto.Marshal(mappings); err != nil { - return oldContent, fmt.Errorf("marshal mappings: %v", err) - } - - return -} - -func RemoveRemoteStorageMapping(oldContent []byte, dir string) (newContent []byte, err error) { - mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent) - if unmarshalErr != nil { - return nil, unmarshalErr - } - - // set the new mapping - delete(mappings.Mappings, dir) - - if newContent, err = proto.Marshal(mappings); err != nil { - return oldContent, fmt.Errorf("marshal mappings: %v", err) - } - - return -} - -func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress string) (mappings *remote_pb.RemoteStorageMapping, readErr error) { - var oldContent []byte - if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE) - return readErr - }); readErr != nil { - return nil, readErr - } - - mappings, readErr = UnmarshalRemoteStorageMappings(oldContent) - if readErr != nil { - return nil, fmt.Errorf("unmarshal mappings: %v", readErr) - } - - return -} - -func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, storageName string) (conf *remote_pb.RemoteConf, readErr error) { +func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error) { var oldContent []byte - if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if readErr = pb.WithFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX) return readErr }); readErr != nil { @@ -199,7 +150,7 @@ func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, return } -func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) { +func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) { mappings, listErr := ReadMountMappings(grpcDialOption, filerAddress) if listErr != nil { diff --git a/weed/filer/filer_remote_storage_test.go b/weed/filer/remote_storage_test.go similarity index 100% rename from weed/filer/filer_remote_storage_test.go rename to weed/filer/remote_storage_test.go diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 729da7d9b..f48c3988c 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -10,7 +10,7 @@ import ( "io" "os" - "github.com/tecbot/gorocksdb" + gorocksdb "github.com/linxGnu/grocksdb" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.go index f6e755b4b..faabcd341 100644 --- a/weed/filer/rocksdb/rocksdb_store_test.go +++ b/weed/filer/rocksdb/rocksdb_store_test.go @@ -1,3 +1,4 @@ +//go:build rocksdb // +build rocksdb package rocksdb @@ -5,7 +6,6 @@ package rocksdb import ( "context" "fmt" - "io/ioutil" "os" "testing" "time" @@ -16,8 +16,7 @@ import ( func TestCreateAndFind(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -35,7 +34,7 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } @@ -70,8 +69,7 @@ func TestCreateAndFind(t *testing.T) { func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -93,8 +91,7 @@ func TestEmptyRoot(t *testing.T) { func BenchmarkInsertEntry(b *testing.B) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench") - defer os.RemoveAll(dir) + dir := b.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filer/rocksdb/rocksdb_ttl.go b/weed/filer/rocksdb/rocksdb_ttl.go index faed22310..7e9643083 100644 --- a/weed/filer/rocksdb/rocksdb_ttl.go +++ b/weed/filer/rocksdb/rocksdb_ttl.go @@ -5,7 +5,7 @@ package rocksdb import ( "time" - "github.com/tecbot/gorocksdb" + gorocksdb "github.com/linxGnu/grocksdb" "github.com/chrislusf/seaweedfs/weed/filer" ) @@ -38,3 +38,8 @@ func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []by func (t *TTLFilter) Name() string { return "TTLFilter" } +func (t *TTLFilter) SetIgnoreSnapshots(value bool) { +} + +func (t *TTLFilter) Destroy() { +} diff --git a/weed/filer/s3iam_conf.go b/weed/filer/s3iam_conf.go index 55c976915..891bf925b 100644 --- a/weed/filer/s3iam_conf.go +++ b/weed/filer/s3iam_conf.go @@ -2,13 +2,12 @@ package filer import ( "bytes" - "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "io" ) -func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error { +func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error { if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil { return err } diff --git a/weed/filer/sqlite/doc.go b/weed/filer/sqlite/doc.go new file mode 100644 index 000000000..833addf54 --- /dev/null +++ b/weed/filer/sqlite/doc.go @@ -0,0 +1,9 @@ +/* + +Package sqlite is for sqlite filer store. + +The referenced "modernc.org/sqlite" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package sqlite diff --git a/weed/filer/sqlite/sqlite_store.go b/weed/filer/sqlite/sqlite_store.go index 6b055e53c..70a4bf390 100644 --- a/weed/filer/sqlite/sqlite_store.go +++ b/weed/filer/sqlite/sqlite_store.go @@ -1,4 +1,6 @@ +//go:build (linux || darwin || windows) && sqlite // +build linux darwin windows +// +build sqlite // limited GOOS due to modernc.org/libc/unistd diff --git a/weed/filer/sqlite/sqlite_store_unsupported.go b/weed/filer/sqlite/sqlite_store_unsupported.go index 803c71afa..351d2e501 100644 --- a/weed/filer/sqlite/sqlite_store_unsupported.go +++ b/weed/filer/sqlite/sqlite_store_unsupported.go @@ -1,4 +1,5 @@ -// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64 +//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64 && !sqlite +// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64,!sqlite // limited GOOS due to modernc.org/libc/unistd diff --git a/weed/filer/store_test/test_suite.go b/weed/filer/store_test/test_suite.go new file mode 100644 index 000000000..ad578442c --- /dev/null +++ b/weed/filer/store_test/test_suite.go @@ -0,0 +1,55 @@ +package store_test + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestFilerStore(t *testing.T, store filer.FilerStore) { + ctx := context.Background() + + store.InsertEntry(ctx, makeEntry(util.FullPath("/"), true)) + store.InsertEntry(ctx, makeEntry(util.FullPath("/a"), true)) + store.InsertEntry(ctx, makeEntry(util.FullPath("/a/b"), true)) + store.InsertEntry(ctx, makeEntry(util.FullPath("/a/b/c"), true)) + for i := 0; i < 2000; i++ { + store.InsertEntry(ctx, makeEntry(util.FullPath(fmt.Sprintf("/a/b/c/f%05d", i)), false)) + } + + { + var counter int + lastFileName, err := store.ListDirectoryEntries(ctx, util.FullPath("/a/b/c"), "", false, 3, func(entry *filer.Entry) bool { + counter++ + return true + }) + assert.Nil(t, err, "list directory") + assert.Equal(t, 3, counter, "directory list counter") + assert.Equal(t, "f00003", lastFileName, "directory list last file") + lastFileName, err = store.ListDirectoryEntries(ctx, util.FullPath("/a/b/c"), lastFileName, false, 1024, func(entry *filer.Entry) bool { + counter++ + return true + }) + assert.Nil(t, err, "list directory") + assert.Equal(t, 1027, counter, "directory list counter") + assert.Equal(t, "f01027", lastFileName, "directory list last file") + } + +} + +func makeEntry(fullPath util.FullPath, isDirectory bool) *filer.Entry { + var mode os.FileMode + if isDirectory { + mode = os.ModeDir + } + return &filer.Entry{ + FullPath: fullPath, + Attr: filer.Attr{ + Mode: mode, + }, + } +} diff --git a/weed/filer/stream.go b/weed/filer/stream.go index ce0264cd3..7da9fd0a0 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -3,6 +3,7 @@ package filer import ( "bytes" "fmt" + "golang.org/x/exp/slices" "io" "math" "sort" @@ -39,11 +40,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool { if len(a) != len(b) { return false } - sort.Slice(a, func(i, j int) bool { - return strings.Compare(a[i].ETag, a[j].ETag) < 0 + slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool { + return strings.Compare(i.ETag, j.ETag) < 0 }) - sort.Slice(b, func(i, j int) bool { - return strings.Compare(b[i].ETag, b[j].ETag) < 0 + slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool { + return strings.Compare(i.ETag, j.ETag) < 0 }) for i := 0; i < len(a); i++ { if a[i].ETag != b[i].ETag { @@ -62,7 +63,7 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { - glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks) + glog.V(4).Infof("start to stream content for chunks: %+v", chunks) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) fileId2Url := make(map[string][]string) @@ -80,11 +81,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ fileId2Url[chunkView.FileId] = urlStrings } + remaining := size for _, chunkView := range chunkViews { - + if offset < chunkView.LogicOffset { + gap := chunkView.LogicOffset - offset + remaining -= gap + glog.V(4).Infof("zero [%d,%d)", offset, chunkView.LogicOffset) + err := writeZero(writer, gap) + if err != nil { + return fmt.Errorf("write zero [%d,%d)", offset, chunkView.LogicOffset) + } + offset = chunkView.LogicOffset + } urlStrings := fileId2Url[chunkView.FileId] start := time.Now() err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) + offset += int64(chunkView.Size) + remaining -= int64(chunkView.Size) stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds()) if err != nil { stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc() @@ -92,6 +105,13 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ } stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc() } + if remaining > 0 { + glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining) + err := writeZero(writer, remaining) + if err != nil { + return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining) + } + } return nil @@ -99,42 +119,59 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ // ---------------- ReadAllReader ---------------------------------- -func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) { +func writeZero(w io.Writer, size int64) (err error) { + zeroPadding := make([]byte, 1024) + var written int + for size > 0 { + if size > 1024 { + written, err = w.Write(zeroPadding) + } else { + written, err = w.Write(zeroPadding[:size]) + } + size -= int64(written) + if err != nil { + return + } + } + return +} - buffer := bytes.Buffer{} +func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) error { lookupFileIdFn := func(fileId string) (targetUrls []string, err error) { return masterClient.LookupFileId(fileId) } - chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64) + chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, int64(len(buffer))) + + idx := 0 for _, chunkView := range chunkViews { urlStrings, err := lookupFileIdFn(chunkView.FileId) if err != nil { glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) - return nil, err + return err } - data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) + n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.Size)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset) if err != nil { - return nil, err + return err } - buffer.Write(data) + idx += n } - return buffer.Bytes(), nil + return nil } // ---------------- ChunkStreamReader ---------------------------------- type ChunkStreamReader struct { - chunkViews []*ChunkView - totalSize int64 - logicOffset int64 - buffer []byte - bufferOffset int64 - bufferLock sync.Mutex - chunk string - lookupFileId wdclient.LookupFileIdFunctionType + chunkViews []*ChunkView + totalSize int64 + logicOffset int64 + buffer []byte + bufferOffset int64 + bufferLock sync.Mutex + chunk string + lookupFileId wdclient.LookupFileIdFunctionType } var _ = io.ReadSeeker(&ChunkStreamReader{}) @@ -143,8 +180,8 @@ var _ = io.ReaderAt(&ChunkStreamReader{}) func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader { chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64) - sort.Slice(chunkViews, func(i, j int) bool { - return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset + slices.SortFunc(chunkViews, func(a, b *ChunkView) bool { + return a.LogicOffset < b.LogicOffset }) var totalSize int64 @@ -206,7 +243,7 @@ func (c *ChunkStreamReader) doRead(p []byte) (n int, err error) { } func (c *ChunkStreamReader) isBufferEmpty() bool { - return len(c.buffer) <= int(c.logicOffset - c.bufferOffset) + return len(c.buffer) <= int(c.logicOffset-c.bufferOffset) } func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) { @@ -261,7 +298,7 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) { } else if currentChunkIndex > 0 { if insideChunk(offset, c.chunkViews[currentChunkIndex]) { // good hit - } else if insideChunk(offset, c.chunkViews[currentChunkIndex-1]){ + } else if insideChunk(offset, c.chunkViews[currentChunkIndex-1]) { currentChunkIndex -= 1 // fmt.Printf("select -1 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId) } else { @@ -297,7 +334,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error { var buffer bytes.Buffer var shouldRetry bool for _, urlString := range urlStrings { - shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) { + shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) { buffer.Write(data) }) if !shouldRetry { diff --git a/weed/filer/tikv/tikv_store.go b/weed/filer/tikv/tikv_store.go index 4a8e8784d..f333af38e 100644 --- a/weed/filer/tikv/tikv_store.go +++ b/weed/filer/tikv/tikv_store.go @@ -1,5 +1,3 @@ -// +build tikv - package tikv import ( @@ -14,7 +12,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/txnkv" ) @@ -27,8 +24,9 @@ func init() { } type TikvStore struct { - client *tikv.KVStore + client *txnkv.Client deleteRangeConcurrency int + onePC bool } // Basic APIs @@ -46,12 +44,13 @@ func (store *TikvStore) Initialize(config util.Configuration, prefix string) err if drc <= 0 { drc = 1 } + store.onePC = config.GetBool(prefix + "enable_1pc") store.deleteRangeConcurrency = drc return store.initialize(pdAddrs) } func (store *TikvStore) initialize(pdAddrs []string) error { - client, err := tikv.NewTxnClient(pdAddrs) + client, err := txnkv.NewClient(pdAddrs) store.client = client return err } @@ -298,6 +297,9 @@ func (store *TikvStore) BeginTransaction(ctx context.Context) (context.Context, if err != nil { return ctx, err } + if store.onePC { + tx.SetEnable1PC(store.onePC) + } return context.WithValue(ctx, "tx", tx), nil } @@ -344,6 +346,9 @@ func (store *TikvStore) getTxn(ctx context.Context) (*TxnWrapper, error) { if err != nil { return nil, err } + if store.onePC { + txn.SetEnable1PC(store.onePC) + } return &TxnWrapper{txn, false}, nil } diff --git a/weed/filer/tikv/tikv_store_kv.go b/weed/filer/tikv/tikv_store_kv.go index 3fed7e045..dcc9acf8c 100644 --- a/weed/filer/tikv/tikv_store_kv.go +++ b/weed/filer/tikv/tikv_store_kv.go @@ -1,5 +1,3 @@ -// +build tikv - package tikv import ( diff --git a/weed/filer/ydb/doc.go b/weed/filer/ydb/doc.go new file mode 100644 index 000000000..6ade3a8d8 --- /dev/null +++ b/weed/filer/ydb/doc.go @@ -0,0 +1,9 @@ +/* + +Package ydb is for YDB filer store. + +The referenced "github.com/ydb-platform/ydb-go-sdk/v3" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package ydb diff --git a/weed/filer/ydb/readme.md b/weed/filer/ydb/readme.md new file mode 100644 index 000000000..b617461fd --- /dev/null +++ b/weed/filer/ydb/readme.md @@ -0,0 +1,27 @@ +## YDB + +database: https://github.com/ydb-platform/ydb + +go driver: https://github.com/ydb-platform/ydb-go-sdk + +options: + +``` +[ydb] +enabled=true +dsn=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db +prefix="seaweedfs" +useBucketPrefix=true +poolSizeLimit=50 +dialTimeOut = 10 +``` + +Authenticate produced with one of next environment variables: + * `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=` — used service account key file by path + * `YDB_ANONYMOUS_CREDENTIALS="1"` — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation + * `YDB_METADATA_CREDENTIALS="1"` — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function + * `YDB_ACCESS_TOKEN_CREDENTIALS=` — used for authenticate to YDB with short-life access token. For example, access token may be IAM token + * `YDB_CONNECTION_STRING="grpcs://endpoint/?database=database"` + + * i test using this dev database: +`make dev_ydb` diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go new file mode 100644 index 000000000..f1db9a143 --- /dev/null +++ b/weed/filer/ydb/ydb_queries.go @@ -0,0 +1,72 @@ +//go:build ydb +// +build ydb + +package ydb + +import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + +const ( + upsertQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $directory AS Utf8; + DECLARE $name AS Utf8; + DECLARE $meta AS String; + DECLARE $expire_at AS Optional; + + UPSERT INTO ` + asql.DEFAULT_TABLE + ` + (dir_hash, name, directory, meta, expire_at) + VALUES + ($dir_hash, $name, $directory, $meta, $expire_at);` + + deleteQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $name AS Utf8; + + DELETE FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND name = $name;` + + findQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $name AS Utf8; + + SELECT meta + FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND name = $name;` + + deleteFolderChildrenQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $directory AS Utf8; + + DELETE FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND directory = $directory;` + + listDirectoryQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $directory AS Utf8; + DECLARE $start_name AS Utf8; + DECLARE $prefix AS Utf8; + DECLARE $limit AS Uint64; + + SELECT name, meta + FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix + ORDER BY name ASC LIMIT $limit;` + + listInclusiveDirectoryQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $directory AS Utf8; + DECLARE $start_name AS Utf8; + DECLARE $prefix AS Utf8; + DECLARE $limit AS Uint64; + + SELECT name, meta + FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix + ORDER BY name ASC LIMIT $limit;` +) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go new file mode 100644 index 000000000..1e3a55a09 --- /dev/null +++ b/weed/filer/ydb/ydb_store.go @@ -0,0 +1,413 @@ +//go:build ydb +// +build ydb + +package ydb + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/sugar" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" + "os" + "path" + "strings" + "sync" + "time" +) + +const ( + defaultDialTimeOut = 10 +) + +var ( + roTX = table.TxControl( + table.BeginTx(table.WithOnlineReadOnly()), + table.CommitTx(), + ) + rwTX = table.DefaultTxControl() +) + +type YdbStore struct { + DB ydb.Connection + dirBuckets string + tablePathPrefix string + SupportBucketTable bool + dbs map[string]bool + dbsLock sync.Mutex +} + +func init() { + filer.Stores = append(filer.Stores, &YdbStore{}) +} + +func (store *YdbStore) GetName() string { + return "ydb" +} + +func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString("filer.options.buckets_folder"), + configuration.GetString(prefix+"dsn"), + configuration.GetString(prefix+"prefix"), + configuration.GetBool(prefix+"useBucketPrefix"), + configuration.GetInt(prefix+"dialTimeOut"), + configuration.GetInt(prefix+"poolSizeLimit"), + ) +} + +func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int) (err error) { + store.dirBuckets = dirBuckets + store.SupportBucketTable = useBucketPrefix + if store.SupportBucketTable { + glog.V(0).Infof("enabled BucketPrefix") + } + store.dbs = make(map[string]bool) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if dialTimeOut == 0 { + dialTimeOut = defaultDialTimeOut + } + opts := []ydb.Option{ + ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second), + environ.WithEnvironCredentials(ctx), + } + if poolSizeLimit > 0 { + opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit)) + } + if dsn == "" { + dsn = os.Getenv("YDB_CONNECTION_STRING") + } + store.DB, err = ydb.Open(ctx, dsn, opts...) + if err != nil { + if store.DB != nil { + _ = store.DB.Close(ctx) + store.DB = nil + } + return fmt.Errorf("can not connect to %s error: %v", dsn, err) + } + + store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix) + if err = sugar.MakeRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { + return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err) + } + + if err = store.createTable(ctx, store.tablePathPrefix); err != nil { + glog.Errorf("createTable %s: %v", store.tablePathPrefix, err) + } + return err +} + +func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) { + var res result.Result + if tx, ok := ctx.Value("tx").(table.Transaction); ok { + res, err = tx.Execute(ctx, *query, params) + if err != nil { + return fmt.Errorf("execute transaction: %v", err) + } + } else { + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + _, res, err = s.Execute(ctx, tc, *query, params) + if err != nil { + return fmt.Errorf("execute statement: %v", err) + } + return nil + }) + } + if err != nil { + return err + } + if res != nil { + defer func() { _ = res.Close() }() + if processResultFunc != nil { + if err = processResultFunc(res); err != nil { + return fmt.Errorf("process result: %v", err) + } + } + } + return err +} + +func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + meta = util.MaybeGzipData(meta) + } + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + fileMeta := FileMeta{util.HashStringToLong(dir), name, *shortDir, meta} + return store.doTxOrDB(ctx, withPragma(tablePathPrefix, upsertQuery), fileMeta.queryParameters(entry.TtlSec), rwTX, nil) +} + +func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + return store.insertOrUpdateEntry(ctx, entry) +} + +func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + return store.insertOrUpdateEntry(ctx, entry) +} + +func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + dir, name := fullpath.DirAndName() + var data []byte + entryFound := false + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, findQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$name", types.UTF8Value(name))) + + err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error { + if !res.NextResultSet(ctx) || !res.HasNextRow() { + return nil + } + for res.NextRow() { + if err = res.ScanNamed(named.OptionalWithDefault("meta", &data)); err != nil { + return fmt.Errorf("scanNamed %s : %v", fullpath, err) + } + entryFound = true + return nil + } + return res.Err() + }) + if err != nil { + return nil, err + } + if !entryFound { + return nil, filer_pb.ErrNotFound + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + return nil, fmt.Errorf("decode %s : %v", fullpath, err) + } + + return entry, nil +} + +func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + dir, name := fullpath.DirAndName() + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, deleteQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$name", types.UTF8Value(name))) + + return store.doTxOrDB(ctx, query, queryParams, rwTX, nil) +} + +func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + dir, _ := fullpath.DirAndName() + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, deleteFolderChildrenQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$directory", types.UTF8Value(*shortDir))) + + return store.doTxOrDB(ctx, query, queryParams, rwTX, nil) +} + +func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) +} + +func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + dir := string(dirPath) + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + var query *string + if includeStartFile { + query = withPragma(tablePathPrefix, listInclusiveDirectoryQuery) + } else { + query = withPragma(tablePathPrefix, listDirectoryQuery) + } + truncated := true + eachEntryFuncIsNotBreake := true + entryCount := int64(0) + for truncated && eachEntryFuncIsNotBreake { + if lastFileName != "" { + startFileName = lastFileName + if includeStartFile { + query = withPragma(tablePathPrefix, listDirectoryQuery) + } + } + restLimit := limit - entryCount + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$directory", types.UTF8Value(*shortDir)), + table.ValueParam("$start_name", types.UTF8Value(startFileName)), + table.ValueParam("$prefix", types.UTF8Value(prefix+"%")), + table.ValueParam("$limit", types.Uint64Value(uint64(restLimit))), + ) + err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error { + var name string + var data []byte + if !res.NextResultSet(ctx) || !res.HasNextRow() { + truncated = false + return nil + } + truncated = res.CurrentResultSet().Truncated() + for res.NextRow() { + if err := res.ScanNamed( + named.OptionalWithDefault("name", &name), + named.OptionalWithDefault("meta", &data)); err != nil { + return fmt.Errorf("list scanNamed %s : %v", dir, err) + } + lastFileName = name + entry := &filer.Entry{ + FullPath: util.NewFullPath(dir, name), + } + if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + return fmt.Errorf("scan decode %s : %v", entry.FullPath, err) + } + if !eachEntryFunc(entry) { + eachEntryFuncIsNotBreake = false + break + } + entryCount += 1 + } + return res.Err() + }) + } + if err != nil { + return lastFileName, err + } + return lastFileName, nil +} + +func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + session, err := store.DB.Table().CreateSession(ctx) + if err != nil { + return ctx, err + } + tx, err := session.BeginTransaction(ctx, table.TxSettings(table.WithSerializableReadWrite())) + if err != nil { + return ctx, err + } + return context.WithValue(ctx, "tx", tx), nil +} + +func (store *YdbStore) CommitTransaction(ctx context.Context) error { + if tx, ok := ctx.Value("tx").(table.Transaction); ok { + _, err := tx.CommitTx(ctx) + return err + } + return nil +} + +func (store *YdbStore) RollbackTransaction(ctx context.Context) error { + if tx, ok := ctx.Value("tx").(table.Transaction); ok { + return tx.Rollback(ctx) + } + return nil +} + +func (store *YdbStore) Shutdown() { + _ = store.DB.Close(context.Background()) +} + +func (store *YdbStore) CanDropWholeBucket() bool { + return store.SupportBucketTable +} + +func (store *YdbStore) OnBucketCreation(bucket string) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if err := store.createTable(context.Background(), + path.Join(store.tablePathPrefix, bucket)); err != nil { + glog.Errorf("createTable %s: %v", bucket, err) + } + + if store.dbs == nil { + return + } + store.dbs[bucket] = true +} + +func (store *YdbStore) OnBucketDeletion(bucket string) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if err := store.deleteTable(context.Background(), + path.Join(store.tablePathPrefix, bucket)); err != nil { + glog.Errorf("deleteTable %s: %v", bucket, err) + } + + if store.dbs == nil { + return + } + delete(store.dbs, bucket) +} + +func (store *YdbStore) createTable(ctx context.Context, prefix string) error { + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), createTableOptions()...) + }) +} + +func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error { + if !store.SupportBucketTable { + return nil + } + if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE)) + }); err != nil { + return err + } + glog.V(4).Infof("deleted table %s", prefix) + + return nil +} + +func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPrefix *string, shortDir *string) { + tablePathPrefix = &store.tablePathPrefix + shortDir = dir + if !store.SupportBucketTable { + return + } + + prefixBuckets := store.dirBuckets + "/" + if strings.HasPrefix(*dir, prefixBuckets) { + // detect bucket + bucketAndDir := (*dir)[len(prefixBuckets):] + var bucket string + if t := strings.Index(bucketAndDir, "/"); t > 0 { + bucket = bucketAndDir[:t] + } else if t < 0 { + bucket = bucketAndDir + } + if bucket == "" { + return + } + + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + tablePathPrefixWithBucket := path.Join(store.tablePathPrefix, bucket) + if _, found := store.dbs[bucket]; !found { + if err := store.createTable(ctx, tablePathPrefixWithBucket); err == nil { + store.dbs[bucket] = true + glog.V(4).Infof("created table %s", tablePathPrefixWithBucket) + } else { + glog.Errorf("createTable %s: %v", tablePathPrefixWithBucket, err) + } + } + tablePathPrefix = &tablePathPrefixWithBucket + } + return +} diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go new file mode 100644 index 000000000..72bbfff42 --- /dev/null +++ b/weed/filer/ydb/ydb_store_kv.go @@ -0,0 +1,75 @@ +//go:build ydb +// +build ydb + +package ydb + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" +) + +func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + fileMeta := FileMeta{dirHash, name, dirStr, value} + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + _, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, upsertQuery), + fileMeta.queryParameters(0)) + if err != nil { + return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) + } + return nil + }) +} + +func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + valueFound := false + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + _, res, err := s.Execute(ctx, roTX, *withPragma(&store.tablePathPrefix, findQuery), + table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name)))) + if err != nil { + return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) + } + defer func() { _ = res.Close() }() + if !res.NextResultSet(ctx) || !res.HasNextRow() { + return nil + } + for res.NextRow() { + if err := res.ScanNamed(named.OptionalWithDefault("meta", &value)); err != nil { + return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name).Name(), err) + } + valueFound = true + return nil + } + return res.Err() + }) + + if !valueFound { + return nil, filer.ErrKvNotFound + } + + return value, nil +} + +func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + _, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, deleteQuery), + table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name)))) + if err != nil { + return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name).Name(), err) + } + return nil + }) + +} diff --git a/weed/filer/ydb/ydb_store_test.go b/weed/filer/ydb/ydb_store_test.go new file mode 100644 index 000000000..cb3c77018 --- /dev/null +++ b/weed/filer/ydb/ydb_store_test.go @@ -0,0 +1,19 @@ +//go:build ydb +// +build ydb + +package ydb + +import ( + "github.com/chrislusf/seaweedfs/weed/filer/store_test" + "testing" +) + +func TestStore(t *testing.T) { + // run "make test_ydb" under docker folder. + // to set up local env + if false { + store := &YdbStore{} + store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50) + store_test.TestFilerStore(t, store) + } +} diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go new file mode 100644 index 000000000..4e5100236 --- /dev/null +++ b/weed/filer/ydb/ydb_types.go @@ -0,0 +1,56 @@ +//go:build ydb +// +build ydb + +package ydb + +import ( + "fmt" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/options" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" +) + +type FileMeta struct { + DirHash int64 `ydb:"type:int64"` + Name string `ydb:"type:utf8"` + Directory string `ydb:"type:utf8"` + Meta []byte `ydb:"type:string"` +} + +type FileMetas []FileMeta + +func (fm *FileMeta) queryParameters(ttlSec int32) *table.QueryParameters { + var expireAtValue types.Value + if ttlSec > 0 { + expireAtValue = types.Uint32Value(uint32(ttlSec)) + } else { + expireAtValue = types.NullValue(types.TypeUint32) + } + return table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)), + table.ValueParam("$directory", types.UTF8Value(fm.Directory)), + table.ValueParam("$name", types.UTF8Value(fm.Name)), + table.ValueParam("$meta", types.StringValue(fm.Meta)), + table.ValueParam("$expire_at", expireAtValue)) +} + +func createTableOptions() []options.CreateTableOption { + columnUnit := options.TimeToLiveUnitSeconds + return []options.CreateTableOption{ + options.WithColumn("dir_hash", types.Optional(types.TypeInt64)), + options.WithColumn("directory", types.Optional(types.TypeUTF8)), + options.WithColumn("name", types.Optional(types.TypeUTF8)), + options.WithColumn("meta", types.Optional(types.TypeString)), + options.WithColumn("expire_at", types.Optional(types.TypeUint32)), + options.WithPrimaryKeyColumn("dir_hash", "name"), + options.WithTimeToLiveSettings(options.TimeToLiveSettings{ + ColumnName: "expire_at", + ColumnUnit: &columnUnit, + Mode: options.TimeToLiveModeValueSinceUnixEpoch}, + ), + } +} +func withPragma(prefix *string, query string) *string { + queryWithPragma := fmt.Sprintf(query, *prefix) + return &queryWithPragma +} diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go deleted file mode 100644 index 9a791e013..000000000 --- a/weed/filesys/dir.go +++ /dev/null @@ -1,618 +0,0 @@ -package filesys - -import ( - "bytes" - "context" - "math" - "os" - "strings" - "syscall" - "time" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" -) - -type Dir struct { - name string - wfs *WFS - entry *filer_pb.Entry - parent *Dir - id uint64 -} - -var _ = fs.Node(&Dir{}) - -var _ = fs.NodeIdentifier(&Dir{}) -var _ = fs.NodeCreater(&Dir{}) -var _ = fs.NodeMknoder(&Dir{}) -var _ = fs.NodeMkdirer(&Dir{}) -var _ = fs.NodeFsyncer(&Dir{}) -var _ = fs.NodeRequestLookuper(&Dir{}) -var _ = fs.HandleReadDirAller(&Dir{}) -var _ = fs.NodeRemover(&Dir{}) -var _ = fs.NodeRenamer(&Dir{}) -var _ = fs.NodeSetattrer(&Dir{}) -var _ = fs.NodeGetxattrer(&Dir{}) -var _ = fs.NodeSetxattrer(&Dir{}) -var _ = fs.NodeRemovexattrer(&Dir{}) -var _ = fs.NodeListxattrer(&Dir{}) -var _ = fs.NodeForgetter(&Dir{}) - -func (dir *Dir) Id() uint64 { - if dir.parent == nil { - return 1 - } - return dir.id -} - -func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { - - entry, err := dir.maybeLoadEntry() - if err != nil { - glog.V(3).Infof("dir Attr %s, err: %+v", dir.FullPath(), err) - return err - } - - // https://github.com/bazil/fuse/issues/196 - attr.Valid = time.Second - attr.Inode = dir.Id() - attr.Mode = os.FileMode(entry.Attributes.FileMode) | os.ModeDir - attr.Mtime = time.Unix(entry.Attributes.Mtime, 0) - attr.Crtime = time.Unix(entry.Attributes.Crtime, 0) - attr.Ctime = time.Unix(entry.Attributes.Crtime, 0) - attr.Atime = time.Unix(entry.Attributes.Mtime, 0) - attr.Gid = entry.Attributes.Gid - attr.Uid = entry.Attributes.Uid - - if dir.FullPath() == dir.wfs.option.FilerMountRootPath { - attr.BlockSize = blockSize - } - - glog.V(4).Infof("dir Attr %s, attr: %+v", dir.FullPath(), attr) - - return nil -} - -func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - - glog.V(4).Infof("dir Getxattr %s", dir.FullPath()) - - entry, err := dir.maybeLoadEntry() - if err != nil { - return err - } - - return getxattr(entry, req, resp) -} - -func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { - // fsync works at OS level - // write the file chunks to the filerGrpcAddress - glog.V(3).Infof("dir %s fsync %+v", dir.FullPath(), req) - - return nil -} - -func (dir *Dir) newFile(name string) fs.Node { - - fileFullPath := util.NewFullPath(dir.FullPath(), name) - fileId := fileFullPath.AsInode() - dir.wfs.handlesLock.Lock() - existingHandle, found := dir.wfs.handles[fileId] - dir.wfs.handlesLock.Unlock() - - if found { - glog.V(4).Infof("newFile found opened file handle: %+v", fileFullPath) - return existingHandle.f - } - return &File{ - Name: name, - dir: dir, - wfs: dir.wfs, - id: fileId, - } -} - -func (dir *Dir) newDirectory(fullpath util.FullPath) fs.Node { - - return &Dir{name: fullpath.Name(), wfs: dir.wfs, parent: dir, id: fullpath.AsInode()} - -} - -func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, - resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { - - exclusive := req.Flags&fuse.OpenExclusive != 0 - isDirectory := req.Mode&os.ModeDir > 0 - - if exclusive || isDirectory { - _, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, exclusive) - if err != nil { - return nil, nil, err - } - } - var node fs.Node - if isDirectory { - node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name)) - return node, node, nil - } - - node = dir.newFile(req.Name) - file := node.(*File) - file.entry = &filer_pb.Entry{ - Name: req.Name, - IsDirectory: req.Mode&os.ModeDir > 0, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(req.Mode &^ dir.wfs.option.Umask), - Uid: req.Uid, - Gid: req.Gid, - Collection: dir.wfs.option.Collection, - Replication: dir.wfs.option.Replication, - TtlSec: dir.wfs.option.TtlSec, - }, - } - file.dirtyMetadata = true - fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid, req.Flags&fuse.OpenWriteOnly > 0) - return file, fh, nil - -} - -func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) { - - _, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false) - - if err != nil { - return nil, err - } - var node fs.Node - node = dir.newFile(req.Name) - return node, nil -} - -func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, exclusive bool) (*filer_pb.CreateEntryRequest, error) { - dirFullPath := dir.FullPath() - request := &filer_pb.CreateEntryRequest{ - Directory: dirFullPath, - Entry: &filer_pb.Entry{ - Name: name, - IsDirectory: mode&os.ModeDir > 0, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(mode &^ dir.wfs.option.Umask), - Uid: uid, - Gid: gid, - Collection: dir.wfs.option.Collection, - Replication: dir.wfs.option.Replication, - TtlSec: dir.wfs.option.TtlSec, - }, - }, - OExcl: exclusive, - Signatures: []int32{dir.wfs.signature}, - } - glog.V(1).Infof("create %s/%s", dirFullPath, name) - - err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - dir.wfs.mapPbIdFromLocalToFiler(request.Entry) - defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) - - if err := filer_pb.CreateEntry(client, request); err != nil { - if strings.Contains(err.Error(), "EEXIST") { - return fuse.EEXIST - } - glog.V(0).Infof("create %s/%s: %v", dirFullPath, name, err) - return fuse.EIO - } - - if err := dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { - glog.Errorf("local InsertEntry dir %s/%s: %v", dirFullPath, name, err) - return fuse.EIO - } - - return nil - }) - return request, err -} - -func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { - - glog.V(4).Infof("mkdir %s: %s", dir.FullPath(), req.Name) - - newEntry := &filer_pb.Entry{ - Name: req.Name, - IsDirectory: true, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(req.Mode &^ dir.wfs.option.Umask), - Uid: req.Uid, - Gid: req.Gid, - }, - } - - dirFullPath := dir.FullPath() - - err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - dir.wfs.mapPbIdFromLocalToFiler(newEntry) - defer dir.wfs.mapPbIdFromFilerToLocal(newEntry) - - request := &filer_pb.CreateEntryRequest{ - Directory: dirFullPath, - Entry: newEntry, - Signatures: []int32{dir.wfs.signature}, - } - - glog.V(1).Infof("mkdir: %v", request) - if err := filer_pb.CreateEntry(client, request); err != nil { - glog.V(0).Infof("mkdir %s/%s: %v", dirFullPath, req.Name, err) - return err - } - - if err := dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { - glog.Errorf("local mkdir dir %s/%s: %v", dirFullPath, req.Name, err) - return fuse.EIO - } - - return nil - }) - - if err == nil { - node := dir.newDirectory(util.NewFullPath(dirFullPath, req.Name)) - - return node, nil - } - - glog.V(0).Infof("mkdir %s/%s: %v", dirFullPath, req.Name, err) - - return nil, fuse.EIO -} - -func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) { - - dirPath := util.FullPath(dir.FullPath()) - // glog.V(4).Infof("dir Lookup %s: %s by %s", dirPath, req.Name, req.Header.String()) - - fullFilePath := dirPath.Child(req.Name) - visitErr := meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath) - if visitErr != nil { - glog.Errorf("dir Lookup %s: %v", dirPath, visitErr) - return nil, fuse.EIO - } - localEntry, cacheErr := dir.wfs.metaCache.FindEntry(context.Background(), fullFilePath) - if cacheErr == filer_pb.ErrNotFound { - return nil, fuse.ENOENT - } - - if localEntry == nil { - // glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath) - entry, err := filer_pb.GetEntry(dir.wfs, fullFilePath) - if err != nil { - glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err) - return nil, fuse.ENOENT - } - localEntry = filer.FromPbEntry(string(dirPath), entry) - } else { - glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath) - } - - if localEntry != nil { - if localEntry.IsDirectory() { - node = dir.newDirectory(fullFilePath) - } else { - node = dir.newFile(req.Name) - } - - // resp.EntryValid = time.Second - resp.Attr.Inode = fullFilePath.AsInode() - resp.Attr.Valid = time.Second - resp.Attr.Size = localEntry.FileSize - resp.Attr.Mtime = localEntry.Attr.Mtime - resp.Attr.Crtime = localEntry.Attr.Crtime - resp.Attr.Mode = localEntry.Attr.Mode - resp.Attr.Gid = localEntry.Attr.Gid - resp.Attr.Uid = localEntry.Attr.Uid - if localEntry.HardLinkCounter > 0 { - resp.Attr.Nlink = uint32(localEntry.HardLinkCounter) - } - - return node, nil - } - - glog.V(4).Infof("not found dir GetEntry %s: %v", fullFilePath, err) - return nil, fuse.ENOENT -} - -func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { - - dirPath := util.FullPath(dir.FullPath()) - glog.V(4).Infof("dir ReadDirAll %s", dirPath) - - processEachEntryFn := func(entry *filer.Entry, isLast bool) { - if entry.IsDirectory() { - dirent := fuse.Dirent{Name: entry.Name(), Type: fuse.DT_Dir, Inode: dirPath.Child(entry.Name()).AsInode()} - ret = append(ret, dirent) - } else { - dirent := fuse.Dirent{Name: entry.Name(), Type: findFileType(uint16(entry.Attr.Mode)), Inode: dirPath.Child(entry.Name()).AsInode()} - ret = append(ret, dirent) - } - } - - if err = meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath); err != nil { - glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) - return nil, fuse.EIO - } - listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool { - processEachEntryFn(entry, false) - return true - }) - if listErr != nil { - glog.Errorf("list meta cache: %v", listErr) - return nil, fuse.EIO - } - - // create proper . and .. directories - ret = append(ret, fuse.Dirent{ - Inode: dirPath.AsInode(), - Name: ".", - Type: fuse.DT_Dir, - }) - - // return the correct parent inode for the mount root - var inode uint64 - if string(dirPath) == dir.wfs.option.FilerMountRootPath { - inode = dir.wfs.option.MountParentInode - } else { - inode = util.FullPath(dir.parent.FullPath()).AsInode() - } - - ret = append(ret, fuse.Dirent{ - Inode: inode, - Name: "..", - Type: fuse.DT_Dir, - }) - - return -} - -func findFileType(mode uint16) fuse.DirentType { - switch mode & (syscall.S_IFMT & 0xffff) { - case syscall.S_IFSOCK: - return fuse.DT_Socket - case syscall.S_IFLNK: - return fuse.DT_Link - case syscall.S_IFREG: - return fuse.DT_File - case syscall.S_IFBLK: - return fuse.DT_Block - case syscall.S_IFDIR: - return fuse.DT_Dir - case syscall.S_IFCHR: - return fuse.DT_Char - case syscall.S_IFIFO: - return fuse.DT_FIFO - } - return fuse.DT_File -} - -func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { - - if !req.Dir { - return dir.removeOneFile(req) - } - - return dir.removeFolder(req) - -} - -func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error { - - dirFullPath := dir.FullPath() - filePath := util.NewFullPath(dirFullPath, req.Name) - entry, err := filer_pb.GetEntry(dir.wfs, filePath) - if err != nil { - return err - } - - // first, ensure the filer store can correctly delete - glog.V(3).Infof("remove file: %v", req) - isDeleteData := entry != nil && entry.HardLinkCounter <= 1 - err = filer_pb.Remove(dir.wfs, dirFullPath, req.Name, isDeleteData, false, false, false, []int32{dir.wfs.signature}) - if err != nil { - glog.V(3).Infof("not found remove file %s: %v", filePath, err) - return fuse.ENOENT - } - - // then, delete meta cache and fsNode cache - if err = dir.wfs.metaCache.DeleteEntry(context.Background(), filePath); err != nil { - glog.V(3).Infof("local DeleteEntry %s: %v", filePath, err) - return fuse.ESTALE - } - - // remove current file handle if any - dir.wfs.handlesLock.Lock() - defer dir.wfs.handlesLock.Unlock() - inodeId := filePath.AsInode() - if fh, ok := dir.wfs.handles[inodeId]; ok { - delete(dir.wfs.handles, inodeId) - fh.isDeleted = true - } - - return nil - -} - -func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error { - - dirFullPath := dir.FullPath() - glog.V(3).Infof("remove directory entry: %v", req) - ignoreRecursiveErr := true // ignore recursion error since the OS should manage it - err := filer_pb.Remove(dir.wfs, dirFullPath, req.Name, true, true, ignoreRecursiveErr, false, []int32{dir.wfs.signature}) - if err != nil { - glog.V(0).Infof("remove %s/%s: %v", dirFullPath, req.Name, err) - if strings.Contains(err.Error(), "non-empty") { - return fuse.EEXIST - } - return fuse.ENOENT - } - - t := util.NewFullPath(dirFullPath, req.Name) - dir.wfs.metaCache.DeleteEntry(context.Background(), t) - - return nil - -} - -func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { - - glog.V(4).Infof("%v dir setattr %+v", dir.FullPath(), req) - - entry, err := dir.maybeLoadEntry() - if err != nil { - return err - } - - if req.Valid.Mode() { - entry.Attributes.FileMode = uint32(req.Mode) - } - - if req.Valid.Uid() { - entry.Attributes.Uid = req.Uid - } - - if req.Valid.Gid() { - entry.Attributes.Gid = req.Gid - } - - if req.Valid.Mtime() { - entry.Attributes.Mtime = req.Mtime.Unix() - } - - return dir.saveEntry(entry) - -} - -func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error { - - glog.V(4).Infof("dir Setxattr %s: %s", dir.FullPath(), req.Name) - - entry, err := dir.maybeLoadEntry() - if err != nil { - return err - } - - if err := setxattr(entry, req); err != nil { - return err - } - - return dir.saveEntry(entry) - -} - -func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error { - - glog.V(4).Infof("dir Removexattr %s: %s", dir.FullPath(), req.Name) - - entry, err := dir.maybeLoadEntry() - if err != nil { - return err - } - - if err := removexattr(entry, req); err != nil { - return err - } - - return dir.saveEntry(entry) - -} - -func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - - glog.V(4).Infof("dir Listxattr %s", dir.FullPath()) - - entry, err := dir.maybeLoadEntry() - if err != nil { - return err - } - - if err := listxattr(entry, req, resp); err != nil { - return err - } - - return nil - -} - -func (dir *Dir) Forget() { - glog.V(4).Infof("Forget dir %s", dir.FullPath()) -} - -func (dir *Dir) maybeLoadEntry() (*filer_pb.Entry, error) { - parentDirPath, name := util.FullPath(dir.FullPath()).DirAndName() - return dir.wfs.maybeLoadEntry(parentDirPath, name) -} - -func (dir *Dir) saveEntry(entry *filer_pb.Entry) error { - - parentDir, name := util.FullPath(dir.FullPath()).DirAndName() - - return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - dir.wfs.mapPbIdFromLocalToFiler(entry) - defer dir.wfs.mapPbIdFromFilerToLocal(entry) - - request := &filer_pb.UpdateEntryRequest{ - Directory: parentDir, - Entry: entry, - Signatures: []int32{dir.wfs.signature}, - } - - glog.V(1).Infof("save dir entry: %v", request) - _, err := client.UpdateEntry(context.Background(), request) - if err != nil { - glog.Errorf("UpdateEntry dir %s/%s: %v", parentDir, name, err) - return fuse.EIO - } - - if err := dir.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { - glog.Errorf("UpdateEntry dir %s/%s: %v", parentDir, name, err) - return fuse.ESTALE - } - - return nil - }) -} - -func (dir *Dir) FullPath() string { - var parts []string - for p := dir; p != nil; p = p.parent { - if strings.HasPrefix(p.name, "/") { - if len(p.name) > 1 { - parts = append(parts, p.name[1:]) - } - } else { - parts = append(parts, p.name) - } - } - - if len(parts) == 0 { - return "/" - } - - var buf bytes.Buffer - for i := len(parts) - 1; i >= 0; i-- { - buf.WriteString("/") - buf.WriteString(parts[i]) - } - return buf.String() -} diff --git a/weed/filesys/dir_link.go b/weed/filesys/dir_link.go deleted file mode 100644 index acdcd2de4..000000000 --- a/weed/filesys/dir_link.go +++ /dev/null @@ -1,160 +0,0 @@ -package filesys - -import ( - "context" - "github.com/chrislusf/seaweedfs/weed/util" - "os" - "syscall" - "time" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" -) - -var _ = fs.NodeLinker(&Dir{}) -var _ = fs.NodeSymlinker(&Dir{}) -var _ = fs.NodeReadlinker(&File{}) - -const ( - HARD_LINK_MARKER = '\x01' -) - -func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (fs.Node, error) { - - oldFile, ok := old.(*File) - if !ok { - glog.Errorf("old node is not a file: %+v", old) - } - - glog.V(4).Infof("Link: %v/%v -> %v/%v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName) - - oldEntry, err := oldFile.maybeLoadEntry(ctx) - if err != nil { - return nil, err - } - - if oldEntry == nil { - return nil, fuse.EIO - } - - // update old file to hardlink mode - if len(oldEntry.HardLinkId) == 0 { - oldEntry.HardLinkId = append(util.RandomBytes(16), HARD_LINK_MARKER) - oldEntry.HardLinkCounter = 1 - } - oldEntry.HardLinkCounter++ - updateOldEntryRequest := &filer_pb.UpdateEntryRequest{ - Directory: oldFile.dir.FullPath(), - Entry: oldEntry, - Signatures: []int32{dir.wfs.signature}, - } - - // CreateLink 1.2 : update new file to hardlink mode - request := &filer_pb.CreateEntryRequest{ - Directory: dir.FullPath(), - Entry: &filer_pb.Entry{ - Name: req.NewName, - IsDirectory: false, - Attributes: oldEntry.Attributes, - Chunks: oldEntry.Chunks, - Extended: oldEntry.Extended, - HardLinkId: oldEntry.HardLinkId, - HardLinkCounter: oldEntry.HardLinkCounter, - }, - Signatures: []int32{dir.wfs.signature}, - } - - // apply changes to the filer, and also apply to local metaCache - err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - dir.wfs.mapPbIdFromLocalToFiler(request.Entry) - defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) - - if err := filer_pb.UpdateEntry(client, updateOldEntryRequest); err != nil { - glog.V(0).Infof("Link %v/%v -> %s/%s: %v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName, err) - return fuse.EIO - } - dir.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry)) - - if err := filer_pb.CreateEntry(client, request); err != nil { - glog.V(0).Infof("Link %v/%v -> %s/%s: %v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName, err) - return fuse.EIO - } - dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) - - return nil - }) - - if err != nil { - return nil, fuse.EIO - } - - // create new file node - newNode := dir.newFile(req.NewName) - newFile := newNode.(*File) - - return newFile, err - -} - -func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) { - - glog.V(4).Infof("Symlink: %v/%v to %v", dir.FullPath(), req.NewName, req.Target) - - request := &filer_pb.CreateEntryRequest{ - Directory: dir.FullPath(), - Entry: &filer_pb.Entry{ - Name: req.NewName, - IsDirectory: false, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32((os.FileMode(0777) | os.ModeSymlink) &^ dir.wfs.option.Umask), - Uid: req.Uid, - Gid: req.Gid, - SymlinkTarget: req.Target, - }, - }, - Signatures: []int32{dir.wfs.signature}, - } - - err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - dir.wfs.mapPbIdFromLocalToFiler(request.Entry) - defer dir.wfs.mapPbIdFromFilerToLocal(request.Entry) - - if err := filer_pb.CreateEntry(client, request); err != nil { - glog.V(0).Infof("symlink %s/%s: %v", dir.FullPath(), req.NewName, err) - return fuse.EIO - } - - dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) - - return nil - }) - - symlink := dir.newFile(req.NewName) - - return symlink, err - -} - -func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return "", err - } - - if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 { - return "", fuse.Errno(syscall.EINVAL) - } - - glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, entry.Attributes.SymlinkTarget) - - return entry.Attributes.SymlinkTarget, nil - -} diff --git a/weed/filesys/dir_rename.go b/weed/filesys/dir_rename.go deleted file mode 100644 index dd76577b0..000000000 --- a/weed/filesys/dir_rename.go +++ /dev/null @@ -1,174 +0,0 @@ -package filesys - -import ( - "context" - "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" - "math" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" -) - -func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error { - - newDir := newDirectory.(*Dir) - - newPath := util.NewFullPath(newDir.FullPath(), req.NewName) - oldPath := util.NewFullPath(dir.FullPath(), req.OldName) - - glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath) - - // find local old entry - oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath) - if err != nil { - glog.Errorf("dir Rename can not find source %s : %v", oldPath, err) - return fuse.ENOENT - } - - // update remote filer - err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - request := &filer_pb.AtomicRenameEntryRequest{ - OldDirectory: dir.FullPath(), - OldName: req.OldName, - NewDirectory: newDir.FullPath(), - NewName: req.NewName, - Signatures: []int32{dir.wfs.signature}, - } - - _, err := client.AtomicRenameEntry(ctx, request) - if err != nil { - glog.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err) - return fuse.EXDEV - } - - return nil - - }) - if err != nil { - glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err) - return fuse.EIO - } - - err = dir.moveEntry(context.Background(), util.FullPath(dir.FullPath()), oldEntry, util.FullPath(newDir.FullPath()), req.NewName) - if err != nil { - glog.V(0).Infof("dir local Rename %s => %s : %v", oldPath, newPath, err) - return fuse.EIO - } - - return nil -} - -func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error { - - oldName := entry.Name() - - oldPath := oldParent.Child(oldName) - newPath := newParent.Child(newName) - if err := dir.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error { - - oldFsNode := NodeWithId(oldPath.AsInode()) - newFsNode := NodeWithId(newPath.AsInode()) - newDirNode, found := dir.wfs.Server.FindInternalNode(NodeWithId(newParent.AsInode())) - var newDir *Dir - if found { - newDir = newDirNode.(*Dir) - } - dir.wfs.Server.InvalidateInternalNode(oldFsNode, newFsNode, func(internalNode fs.Node) { - if file, ok := internalNode.(*File); ok { - glog.V(4).Infof("internal file node %s", oldParent.Child(oldName)) - file.Name = newName - file.id = uint64(newFsNode) - if found { - file.dir = newDir - } - } - if dir, ok := internalNode.(*Dir); ok { - glog.V(4).Infof("internal dir node %s", oldParent.Child(oldName)) - dir.name = newName - dir.id = uint64(newFsNode) - if found { - dir.parent = newDir - } - } - }) - - // change file handle - inodeId := oldPath.AsInode() - dir.wfs.handlesLock.Lock() - if existingHandle, found := dir.wfs.handles[inodeId]; found && existingHandle == nil { - glog.V(4).Infof("opened file handle %s => %s", oldPath, newPath) - delete(dir.wfs.handles, inodeId) - dir.wfs.handles[newPath.AsInode()] = existingHandle - } - dir.wfs.handlesLock.Unlock() - - if entry.IsDirectory() { - if err := dir.moveFolderSubEntries(ctx, oldParent, oldName, newParent, newName); err != nil { - return err - } - } - return nil - }); err != nil { - return fmt.Errorf("fail to move %s => %s: %v", oldPath, newPath, err) - } - - return nil -} - -func (dir *Dir) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, oldName string, newParent util.FullPath, newName string) error { - - currentDirPath := oldParent.Child(oldName) - newDirPath := newParent.Child(newName) - - glog.V(1).Infof("moving folder %s => %s", currentDirPath, newDirPath) - - var moveErr error - listErr := dir.wfs.metaCache.ListDirectoryEntries(ctx, currentDirPath, "", false, int64(math.MaxInt32), func(item *filer.Entry) bool { - moveErr = dir.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name()) - if moveErr != nil { - return false - } - return true - }) - if listErr != nil { - return listErr - } - if moveErr != nil { - return moveErr - } - - return nil -} - -func (dir *Dir) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error { - - newPath := newParent.Child(newName) - oldPath := oldParent.Child(entry.Name()) - - entry.FullPath = newPath - if err := dir.wfs.metaCache.InsertEntry(ctx, entry); err != nil { - glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err) - return fuse.EIO - } - - if moveFolderSubEntries != nil { - if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil { - return moveChildrenErr - } - } - - if err := dir.wfs.metaCache.DeleteEntry(ctx, oldPath); err != nil { - glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err) - return fuse.EIO - } - - return nil -} diff --git a/weed/filesys/dirty_page_interval.go b/weed/filesys/dirty_page_interval.go deleted file mode 100644 index 304793340..000000000 --- a/weed/filesys/dirty_page_interval.go +++ /dev/null @@ -1,222 +0,0 @@ -package filesys - -import ( - "io" - - "github.com/chrislusf/seaweedfs/weed/util" -) - -type IntervalNode struct { - Data []byte - Offset int64 - Size int64 - Next *IntervalNode -} - -type IntervalLinkedList struct { - Head *IntervalNode - Tail *IntervalNode -} - -type ContinuousIntervals struct { - lists []*IntervalLinkedList -} - -func (list *IntervalLinkedList) Offset() int64 { - return list.Head.Offset -} -func (list *IntervalLinkedList) Size() int64 { - return list.Tail.Offset + list.Tail.Size - list.Head.Offset -} -func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) { - // glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size) - list.Tail.Next = node - list.Tail = node -} -func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) { - // glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size) - node.Next = list.Head - list.Head = node -} - -func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) { - t := list.Head - for { - - nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size) - if nodeStart < nodeStop { - // glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop) - copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset]) - } - - if t.Next == nil { - break - } - t = t.Next - } -} - -func (c *ContinuousIntervals) TotalSize() (total int64) { - for _, list := range c.lists { - total += list.Size() - } - return -} - -func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList { - var nodes []*IntervalNode - for t := list.Head; t != nil; t = t.Next { - nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size) - if nodeStart >= nodeStop { - // skip non overlapping IntervalNode - continue - } - nodes = append(nodes, &IntervalNode{ - Data: t.Data[nodeStart-t.Offset : nodeStop-t.Offset], - Offset: nodeStart, - Size: nodeStop - nodeStart, - Next: nil, - }) - } - for i := 1; i < len(nodes); i++ { - nodes[i-1].Next = nodes[i] - } - return &IntervalLinkedList{ - Head: nodes[0], - Tail: nodes[len(nodes)-1], - } -} - -func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { - - interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))} - - // append to the tail and return - if len(c.lists) == 1 { - lastSpan := c.lists[0] - if lastSpan.Tail.Offset+lastSpan.Tail.Size == offset { - lastSpan.addNodeToTail(interval) - return - } - } - - var newLists []*IntervalLinkedList - for _, list := range c.lists { - // if list is to the left of new interval, add to the new list - if list.Tail.Offset+list.Tail.Size <= interval.Offset { - newLists = append(newLists, list) - } - // if list is to the right of new interval, add to the new list - if interval.Offset+interval.Size <= list.Head.Offset { - newLists = append(newLists, list) - } - // if new interval overwrite the right part of the list - if list.Head.Offset < interval.Offset && interval.Offset < list.Tail.Offset+list.Tail.Size { - // create a new list of the left part of existing list - newLists = append(newLists, subList(list, list.Offset(), interval.Offset)) - } - // if new interval overwrite the left part of the list - if list.Head.Offset < interval.Offset+interval.Size && interval.Offset+interval.Size < list.Tail.Offset+list.Tail.Size { - // create a new list of the right part of existing list - newLists = append(newLists, subList(list, interval.Offset+interval.Size, list.Tail.Offset+list.Tail.Size)) - } - // skip anything that is fully overwritten by the new interval - } - - c.lists = newLists - // add the new interval to the lists, connecting neighbor lists - var prevList, nextList *IntervalLinkedList - - for _, list := range c.lists { - if list.Head.Offset == interval.Offset+interval.Size { - nextList = list - break - } - } - - for _, list := range c.lists { - if list.Head.Offset+list.Size() == offset { - list.addNodeToTail(interval) - prevList = list - break - } - } - - if prevList != nil && nextList != nil { - // glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size) - prevList.Tail.Next = nextList.Head - prevList.Tail = nextList.Tail - c.removeList(nextList) - } else if nextList != nil { - // add to head was not done when checking - nextList.addNodeToHead(interval) - } - if prevList == nil && nextList == nil { - c.lists = append(c.lists, &IntervalLinkedList{ - Head: interval, - Tail: interval, - }) - } - - return -} - -func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList { - var maxSize int64 - maxIndex := -1 - for k, list := range c.lists { - if maxSize <= list.Size() { - maxSize = list.Size() - maxIndex = k - } - } - if maxSize <= 0 { - return nil - } - - t := c.lists[maxIndex] - c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...) - return t - -} - -func (c *ContinuousIntervals) removeList(target *IntervalLinkedList) { - index := -1 - for k, list := range c.lists { - if list.Offset() == target.Offset() { - index = k - } - } - if index < 0 { - return - } - - c.lists = append(c.lists[0:index], c.lists[index+1:]...) - -} - -func (c *ContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxStop int64) { - for _, list := range c.lists { - start := max(startOffset, list.Offset()) - stop := min(startOffset+int64(len(data)), list.Offset()+list.Size()) - if start < stop { - list.ReadData(data[start-startOffset:], start, stop) - maxStop = max(maxStop, stop) - } - } - return -} - -func (l *IntervalLinkedList) ToReader() io.Reader { - var readers []io.Reader - t := l.Head - readers = append(readers, util.NewBytesReader(t.Data)) - for t.Next != nil { - t = t.Next - readers = append(readers, util.NewBytesReader(t.Data)) - } - if len(readers) == 1 { - return readers[0] - } - return io.MultiReader(readers...) -} diff --git a/weed/filesys/dirty_page_interval_test.go b/weed/filesys/dirty_page_interval_test.go deleted file mode 100644 index d02ad27fd..000000000 --- a/weed/filesys/dirty_page_interval_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package filesys - -import ( - "bytes" - "math/rand" - "testing" -) - -func TestContinuousIntervals_AddIntervalAppend(t *testing.T) { - - c := &ContinuousIntervals{} - - // 25, 25, 25 - c.AddInterval(getBytes(25, 3), 0) - // _, _, 23, 23, 23, 23 - c.AddInterval(getBytes(23, 4), 2) - - expectedData(t, c, 0, 25, 25, 23, 23, 23, 23) - -} - -func TestContinuousIntervals_AddIntervalInnerOverwrite(t *testing.T) { - - c := &ContinuousIntervals{} - - // 25, 25, 25, 25, 25 - c.AddInterval(getBytes(25, 5), 0) - // _, _, 23, 23 - c.AddInterval(getBytes(23, 2), 2) - - expectedData(t, c, 0, 25, 25, 23, 23, 25) - -} - -func TestContinuousIntervals_AddIntervalFullOverwrite(t *testing.T) { - - c := &ContinuousIntervals{} - - // 1, - c.AddInterval(getBytes(1, 1), 0) - // _, 2, - c.AddInterval(getBytes(2, 1), 1) - // _, _, 3, 3, 3 - c.AddInterval(getBytes(3, 3), 2) - // _, _, _, 4, 4, 4 - c.AddInterval(getBytes(4, 3), 3) - - expectedData(t, c, 0, 1, 2, 3, 4, 4, 4) - -} - -func TestContinuousIntervals_RealCase1(t *testing.T) { - - c := &ContinuousIntervals{} - - // 25, - c.AddInterval(getBytes(25, 1), 0) - // _, _, _, _, 23, 23 - c.AddInterval(getBytes(23, 2), 4) - // _, _, _, 24, 24, 24, 24 - c.AddInterval(getBytes(24, 4), 3) - - // _, 22, 22 - c.AddInterval(getBytes(22, 2), 1) - - expectedData(t, c, 0, 25, 22, 22, 24, 24, 24, 24) - -} - -func TestRandomWrites(t *testing.T) { - - c := &ContinuousIntervals{} - - data := make([]byte, 1024) - - for i := 0; i < 1024; i++ { - - start, stop := rand.Intn(len(data)), rand.Intn(len(data)) - if start > stop { - start, stop = stop, start - } - - rand.Read(data[start : stop+1]) - - c.AddInterval(data[start:stop+1], int64(start)) - - expectedData(t, c, 0, data...) - - } - -} - -func expectedData(t *testing.T, c *ContinuousIntervals, offset int, data ...byte) { - start, stop := int64(offset), int64(offset+len(data)) - for _, list := range c.lists { - nodeStart, nodeStop := max(start, list.Head.Offset), min(stop, list.Head.Offset+list.Size()) - if nodeStart < nodeStop { - buf := make([]byte, nodeStop-nodeStart) - list.ReadData(buf, nodeStart, nodeStop) - if bytes.Compare(buf, data[nodeStart-start:nodeStop-start]) != 0 { - t.Errorf("expected %v actual %v", data[nodeStart-start:nodeStop-start], buf) - } - } - } -} - -func getBytes(content byte, length int) []byte { - data := make([]byte, length) - for i := 0; i < length; i++ { - data[i] = content - } - return data -} diff --git a/weed/filesys/dirty_pages.go b/weed/filesys/dirty_pages.go deleted file mode 100644 index 8505323ef..000000000 --- a/weed/filesys/dirty_pages.go +++ /dev/null @@ -1,10 +0,0 @@ -package filesys - -type DirtyPages interface { - AddPage(offset int64, data []byte) - FlushData() error - ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) - GetStorageOptions() (collection, replication string) - SetWriteOnly(writeOnly bool) - GetWriteOnly() (writeOnly bool) -} diff --git a/weed/filesys/dirty_pages_continuous.go b/weed/filesys/dirty_pages_continuous.go deleted file mode 100644 index b7514a2eb..000000000 --- a/weed/filesys/dirty_pages_continuous.go +++ /dev/null @@ -1,160 +0,0 @@ -package filesys - -import ( - "bytes" - "fmt" - "io" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" -) - -type ContinuousDirtyPages struct { - intervals *ContinuousIntervals - f *File - writeOnly bool - writeWaitGroup sync.WaitGroup - chunkAddLock sync.Mutex - lastErr error - collection string - replication string -} - -func newContinuousDirtyPages(file *File, writeOnly bool) *ContinuousDirtyPages { - dirtyPages := &ContinuousDirtyPages{ - intervals: &ContinuousIntervals{}, - f: file, - writeOnly: writeOnly, - } - return dirtyPages -} - -func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) { - - glog.V(4).Infof("%s AddPage [%d,%d)", pages.f.fullpath(), offset, offset+int64(len(data))) - - if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) { - // this is more than what buffer can hold. - pages.flushAndSave(offset, data) - } - - pages.intervals.AddInterval(data, offset) - - if pages.intervals.TotalSize() >= pages.f.wfs.option.ChunkSizeLimit { - pages.saveExistingLargestPageToStorage() - } - - return -} - -func (pages *ContinuousDirtyPages) flushAndSave(offset int64, data []byte) { - - // flush existing - pages.saveExistingPagesToStorage() - - // flush the new page - pages.saveToStorage(bytes.NewReader(data), offset, int64(len(data))) - - return -} - -func (pages *ContinuousDirtyPages) FlushData() error { - - pages.saveExistingPagesToStorage() - pages.writeWaitGroup.Wait() - if pages.lastErr != nil { - return fmt.Errorf("flush data: %v", pages.lastErr) - } - return nil -} - -func (pages *ContinuousDirtyPages) saveExistingPagesToStorage() { - for pages.saveExistingLargestPageToStorage() { - } -} - -func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedData bool) { - - maxList := pages.intervals.RemoveLargestIntervalLinkedList() - if maxList == nil { - return false - } - - entry := pages.f.getEntry() - if entry == nil { - return false - } - - fileSize := int64(entry.Attributes.FileSize) - - chunkSize := min(maxList.Size(), fileSize-maxList.Offset()) - if chunkSize == 0 { - return false - } - - pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize) - - return true -} - -func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) { - - mtime := time.Now().UnixNano() - pages.writeWaitGroup.Add(1) - writer := func() { - defer pages.writeWaitGroup.Done() - - reader = io.LimitReader(reader, size) - chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset) - if err != nil { - glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err) - pages.lastErr = err - return - } - chunk.Mtime = mtime - pages.collection, pages.replication = collection, replication - pages.chunkAddLock.Lock() - defer pages.chunkAddLock.Unlock() - pages.f.addChunks([]*filer_pb.FileChunk{chunk}) - glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size) - } - - if pages.f.wfs.concurrentWriters != nil { - pages.f.wfs.concurrentWriters.Execute(writer) - } else { - go writer() - } -} - -func max(x, y int64) int64 { - if x > y { - return x - } - return y -} -func min(x, y int64) int64 { - if x < y { - return x - } - return y -} - -func (pages *ContinuousDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { - return pages.intervals.ReadDataAt(data, startOffset) -} - -func (pages *ContinuousDirtyPages) GetStorageOptions() (collection, replication string) { - return pages.collection, pages.replication -} - -func (pages *ContinuousDirtyPages) SetWriteOnly(writeOnly bool) { - if pages.writeOnly { - pages.writeOnly = writeOnly - } -} - -func (pages *ContinuousDirtyPages) GetWriteOnly() (writeOnly bool) { - return pages.writeOnly -} diff --git a/weed/filesys/dirty_pages_temp_file.go b/weed/filesys/dirty_pages_temp_file.go deleted file mode 100644 index 9fa7c0c8e..000000000 --- a/weed/filesys/dirty_pages_temp_file.go +++ /dev/null @@ -1,157 +0,0 @@ -package filesys - -import ( - "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "io" - "os" - "sync" - "time" -) - -type TempFileDirtyPages struct { - f *File - tf *os.File - writtenIntervals *WrittenContinuousIntervals - writeOnly bool - writeWaitGroup sync.WaitGroup - pageAddLock sync.Mutex - chunkAddLock sync.Mutex - lastErr error - collection string - replication string -} - -func newTempFileDirtyPages(file *File, writeOnly bool) *TempFileDirtyPages { - - tempFile := &TempFileDirtyPages{ - f: file, - writeOnly: writeOnly, - writtenIntervals: &WrittenContinuousIntervals{}, - } - - return tempFile -} - -func (pages *TempFileDirtyPages) AddPage(offset int64, data []byte) { - - pages.pageAddLock.Lock() - defer pages.pageAddLock.Unlock() - - if pages.tf == nil { - tf, err := os.CreateTemp(pages.f.wfs.option.getTempFilePageDir(), "") - if err != nil { - glog.Errorf("create temp file: %v", err) - pages.lastErr = err - return - } - pages.tf = tf - pages.writtenIntervals.tempFile = tf - pages.writtenIntervals.lastOffset = 0 - } - - writtenOffset := pages.writtenIntervals.lastOffset - dataSize := int64(len(data)) - - // glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+dataSize) - - if _, err := pages.tf.WriteAt(data, writtenOffset); err != nil { - pages.lastErr = err - } else { - pages.writtenIntervals.AddInterval(writtenOffset, len(data), offset) - pages.writtenIntervals.lastOffset += dataSize - } - - // pages.writtenIntervals.debug() - - return -} - -func (pages *TempFileDirtyPages) FlushData() error { - - pages.saveExistingPagesToStorage() - pages.writeWaitGroup.Wait() - if pages.lastErr != nil { - return fmt.Errorf("flush data: %v", pages.lastErr) - } - pages.pageAddLock.Lock() - defer pages.pageAddLock.Unlock() - if pages.tf != nil { - - pages.writtenIntervals.tempFile = nil - pages.writtenIntervals.lists = nil - - pages.tf.Close() - os.Remove(pages.tf.Name()) - pages.tf = nil - } - return nil -} - -func (pages *TempFileDirtyPages) saveExistingPagesToStorage() { - - pageSize := pages.f.wfs.option.ChunkSizeLimit - - // glog.V(4).Infof("%v saveExistingPagesToStorage %d lists", pages.f.Name, len(pages.writtenIntervals.lists)) - - for _, list := range pages.writtenIntervals.lists { - listStopOffset := list.Offset() + list.Size() - for uploadedOffset := int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize { - start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize) - if start >= stop { - continue - } - // glog.V(4).Infof("uploading %v [%d,%d) %d/%d", pages.f.Name, start, stop, i, len(pages.writtenIntervals.lists)) - pages.saveToStorage(list.ToReader(start, stop), start, stop-start) - } - } - -} - -func (pages *TempFileDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) { - - mtime := time.Now().UnixNano() - pages.writeWaitGroup.Add(1) - writer := func() { - defer pages.writeWaitGroup.Done() - - reader = io.LimitReader(reader, size) - chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset) - if err != nil { - glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err) - pages.lastErr = err - return - } - chunk.Mtime = mtime - pages.collection, pages.replication = collection, replication - pages.chunkAddLock.Lock() - defer pages.chunkAddLock.Unlock() - pages.f.addChunks([]*filer_pb.FileChunk{chunk}) - glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.f.fullpath(), chunk.FileId, offset, offset+size) - } - - if pages.f.wfs.concurrentWriters != nil { - pages.f.wfs.concurrentWriters.Execute(writer) - } else { - go writer() - } -} - -func (pages *TempFileDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { - return pages.writtenIntervals.ReadDataAt(data, startOffset) -} - -func (pages *TempFileDirtyPages) GetStorageOptions() (collection, replication string) { - return pages.collection, pages.replication -} - -func (pages *TempFileDirtyPages) SetWriteOnly(writeOnly bool) { - if pages.writeOnly { - pages.writeOnly = writeOnly - } -} - -func (pages *TempFileDirtyPages) GetWriteOnly() (writeOnly bool) { - return pages.writeOnly -} diff --git a/weed/filesys/dirty_pages_temp_interval.go b/weed/filesys/dirty_pages_temp_interval.go deleted file mode 100644 index 42c4b5a3b..000000000 --- a/weed/filesys/dirty_pages_temp_interval.go +++ /dev/null @@ -1,289 +0,0 @@ -package filesys - -import ( - "io" - "log" - "os" -) - -type WrittenIntervalNode struct { - DataOffset int64 - TempOffset int64 - Size int64 - Next *WrittenIntervalNode -} - -type WrittenIntervalLinkedList struct { - tempFile *os.File - Head *WrittenIntervalNode - Tail *WrittenIntervalNode -} - -type WrittenContinuousIntervals struct { - tempFile *os.File - lastOffset int64 - lists []*WrittenIntervalLinkedList -} - -func (list *WrittenIntervalLinkedList) Offset() int64 { - return list.Head.DataOffset -} -func (list *WrittenIntervalLinkedList) Size() int64 { - return list.Tail.DataOffset + list.Tail.Size - list.Head.DataOffset -} -func (list *WrittenIntervalLinkedList) addNodeToTail(node *WrittenIntervalNode) { - // glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size) - if list.Tail.TempOffset+list.Tail.Size == node.TempOffset { - // already connected - list.Tail.Size += node.Size - } else { - list.Tail.Next = node - list.Tail = node - } -} -func (list *WrittenIntervalLinkedList) addNodeToHead(node *WrittenIntervalNode) { - // glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size) - node.Next = list.Head - list.Head = node -} - -func (list *WrittenIntervalLinkedList) ReadData(buf []byte, start, stop int64) { - t := list.Head - for { - - nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) - if nodeStart < nodeStop { - // glog.V(4).Infof("copying start=%d stop=%d t=[%d,%d) => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.DataOffset, t.DataOffset+t.Size, len(buf), nodeStart, nodeStop) - list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset+nodeStart-t.DataOffset) - } - - if t.Next == nil { - break - } - t = t.Next - } -} - -func (c *WrittenContinuousIntervals) TotalSize() (total int64) { - for _, list := range c.lists { - total += list.Size() - } - return -} - -func (list *WrittenIntervalLinkedList) subList(start, stop int64) *WrittenIntervalLinkedList { - var nodes []*WrittenIntervalNode - for t := list.Head; t != nil; t = t.Next { - nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) - if nodeStart >= nodeStop { - // skip non overlapping WrittenIntervalNode - continue - } - nodes = append(nodes, &WrittenIntervalNode{ - TempOffset: t.TempOffset + nodeStart - t.DataOffset, - DataOffset: nodeStart, - Size: nodeStop - nodeStart, - Next: nil, - }) - } - for i := 1; i < len(nodes); i++ { - nodes[i-1].Next = nodes[i] - } - return &WrittenIntervalLinkedList{ - tempFile: list.tempFile, - Head: nodes[0], - Tail: nodes[len(nodes)-1], - } -} - -func (c *WrittenContinuousIntervals) debug() { - log.Printf("++") - for _, l := range c.lists { - log.Printf("++++") - for t := l.Head; ; t = t.Next { - log.Printf("[%d,%d) => [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size) - if t.Next == nil { - break - } - } - log.Printf("----") - } - log.Printf("--") -} - -func (c *WrittenContinuousIntervals) AddInterval(tempOffset int64, dataSize int, dataOffset int64) { - - interval := &WrittenIntervalNode{DataOffset: dataOffset, TempOffset: tempOffset, Size: int64(dataSize)} - - // append to the tail and return - if len(c.lists) == 1 { - lastSpan := c.lists[0] - if lastSpan.Tail.DataOffset+lastSpan.Tail.Size == dataOffset { - lastSpan.addNodeToTail(interval) - return - } - } - - var newLists []*WrittenIntervalLinkedList - for _, list := range c.lists { - // if list is to the left of new interval, add to the new list - if list.Tail.DataOffset+list.Tail.Size <= interval.DataOffset { - newLists = append(newLists, list) - } - // if list is to the right of new interval, add to the new list - if interval.DataOffset+interval.Size <= list.Head.DataOffset { - newLists = append(newLists, list) - } - // if new interval overwrite the right part of the list - if list.Head.DataOffset < interval.DataOffset && interval.DataOffset < list.Tail.DataOffset+list.Tail.Size { - // create a new list of the left part of existing list - newLists = append(newLists, list.subList(list.Offset(), interval.DataOffset)) - } - // if new interval overwrite the left part of the list - if list.Head.DataOffset < interval.DataOffset+interval.Size && interval.DataOffset+interval.Size < list.Tail.DataOffset+list.Tail.Size { - // create a new list of the right part of existing list - newLists = append(newLists, list.subList(interval.DataOffset+interval.Size, list.Tail.DataOffset+list.Tail.Size)) - } - // skip anything that is fully overwritten by the new interval - } - - c.lists = newLists - // add the new interval to the lists, connecting neighbor lists - var prevList, nextList *WrittenIntervalLinkedList - - for _, list := range c.lists { - if list.Head.DataOffset == interval.DataOffset+interval.Size { - nextList = list - break - } - } - - for _, list := range c.lists { - if list.Head.DataOffset+list.Size() == dataOffset { - list.addNodeToTail(interval) - prevList = list - break - } - } - - if prevList != nil && nextList != nil { - // glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size) - prevList.Tail.Next = nextList.Head - prevList.Tail = nextList.Tail - c.removeList(nextList) - } else if nextList != nil { - // add to head was not done when checking - nextList.addNodeToHead(interval) - } - if prevList == nil && nextList == nil { - c.lists = append(c.lists, &WrittenIntervalLinkedList{ - tempFile: c.tempFile, - Head: interval, - Tail: interval, - }) - } - - return -} - -func (c *WrittenContinuousIntervals) RemoveLargestIntervalLinkedList() *WrittenIntervalLinkedList { - var maxSize int64 - maxIndex := -1 - for k, list := range c.lists { - if maxSize <= list.Size() { - maxSize = list.Size() - maxIndex = k - } - } - if maxSize <= 0 { - return nil - } - - t := c.lists[maxIndex] - t.tempFile = c.tempFile - c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...) - return t - -} - -func (c *WrittenContinuousIntervals) removeList(target *WrittenIntervalLinkedList) { - index := -1 - for k, list := range c.lists { - if list.Offset() == target.Offset() { - index = k - } - } - if index < 0 { - return - } - - c.lists = append(c.lists[0:index], c.lists[index+1:]...) - -} - -func (c *WrittenContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxStop int64) { - for _, list := range c.lists { - start := max(startOffset, list.Offset()) - stop := min(startOffset+int64(len(data)), list.Offset()+list.Size()) - if start < stop { - list.ReadData(data[start-startOffset:], start, stop) - maxStop = max(maxStop, stop) - } - } - return -} - -func (l *WrittenIntervalLinkedList) ToReader(start int64, stop int64) io.Reader { - // TODO: optimize this to avoid another loop - var readers []io.Reader - for t := l.Head; ; t = t.Next { - startOffset, stopOffset := max(t.DataOffset, start), min(t.DataOffset+t.Size, stop) - if startOffset < stopOffset { - // glog.V(4).Infof("ToReader read [%d,%d) from [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size) - readers = append(readers, newFileSectionReader(l.tempFile, startOffset-t.DataOffset+t.TempOffset, startOffset, stopOffset-startOffset)) - } - if t.Next == nil { - break - } - } - if len(readers) == 1 { - return readers[0] - } - return io.MultiReader(readers...) -} - -type FileSectionReader struct { - file *os.File - tempStartOffset int64 - Offset int64 - dataStart int64 - dataStop int64 -} - -var _ = io.Reader(&FileSectionReader{}) - -func newFileSectionReader(tempfile *os.File, offset int64, dataOffset int64, size int64) *FileSectionReader { - return &FileSectionReader{ - file: tempfile, - tempStartOffset: offset, - Offset: offset, - dataStart: dataOffset, - dataStop: dataOffset + size, - } -} - -func (f *FileSectionReader) Read(p []byte) (n int, err error) { - remaining := (f.dataStop - f.dataStart) - (f.Offset - f.tempStartOffset) - if remaining <= 0 { - return 0, io.EOF - } - dataLen := min(remaining, int64(len(p))) - // glog.V(4).Infof("reading [%d,%d) from %v [%d,%d)/[%d,%d) %d", f.Offset-f.tempStartOffset+f.dataStart, f.Offset-f.tempStartOffset+f.dataStart+dataLen, f.file.Name(), f.Offset, f.Offset+dataLen, f.tempStartOffset, f.tempStartOffset+f.dataStop-f.dataStart, f.dataStop-f.dataStart) - n, err = f.file.ReadAt(p[:dataLen], f.Offset) - if n > 0 { - f.Offset += int64(n) - } else { - err = io.EOF - } - return -} diff --git a/weed/filesys/file.go b/weed/filesys/file.go deleted file mode 100644 index b990b20d1..000000000 --- a/weed/filesys/file.go +++ /dev/null @@ -1,389 +0,0 @@ -package filesys - -import ( - "context" - "os" - "sort" - "time" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" -) - -const blockSize = 512 - -var _ = fs.Node(&File{}) -var _ = fs.NodeIdentifier(&File{}) -var _ = fs.NodeOpener(&File{}) -var _ = fs.NodeFsyncer(&File{}) -var _ = fs.NodeSetattrer(&File{}) -var _ = fs.NodeGetxattrer(&File{}) -var _ = fs.NodeSetxattrer(&File{}) -var _ = fs.NodeRemovexattrer(&File{}) -var _ = fs.NodeListxattrer(&File{}) -var _ = fs.NodeForgetter(&File{}) - -type File struct { - Name string - dir *Dir - wfs *WFS - entry *filer_pb.Entry - isOpen int - dirtyMetadata bool - id uint64 -} - -func (file *File) fullpath() util.FullPath { - return util.NewFullPath(file.dir.FullPath(), file.Name) -} - -func (file *File) Id() uint64 { - return file.id -} - -func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { - - glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - if entry == nil { - return fuse.ENOENT - } - - attr.Inode = file.Id() - attr.Valid = time.Second - attr.Mode = os.FileMode(entry.Attributes.FileMode) - attr.Size = filer.FileSize(entry) - if file.isOpen > 0 { - attr.Size = entry.Attributes.FileSize - glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size) - } - attr.Crtime = time.Unix(entry.Attributes.Crtime, 0) - attr.Mtime = time.Unix(entry.Attributes.Mtime, 0) - attr.Gid = entry.Attributes.Gid - attr.Uid = entry.Attributes.Uid - attr.Blocks = attr.Size/blockSize + 1 - attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit) - if entry.HardLinkCounter > 0 { - attr.Nlink = uint32(entry.HardLinkCounter) - } - - return nil - -} - -func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - - // glog.V(4).Infof("file Getxattr %s", file.fullpath()) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - return getxattr(entry, req, resp) -} - -func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { - - glog.V(4).Infof("file %v open %+v", file.fullpath(), req) - - handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid, req.Flags&fuse.OpenWriteOnly > 0) - - resp.Handle = fuse.HandleID(handle.handle) - - glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle) - - return handle, nil - -} - -func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { - - glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - if req.Valid.Size() { - - glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks)) - if req.Size < filer.FileSize(entry) { - // fmt.Printf("truncate %v \n", fullPath) - var chunks []*filer_pb.FileChunk - var truncatedChunks []*filer_pb.FileChunk - for _, chunk := range entry.Chunks { - int64Size := int64(chunk.Size) - if chunk.Offset+int64Size > int64(req.Size) { - // this chunk is truncated - int64Size = int64(req.Size) - chunk.Offset - if int64Size > 0 { - chunks = append(chunks, chunk) - glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size) - chunk.Size = uint64(int64Size) - } else { - glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString()) - truncatedChunks = append(truncatedChunks, chunk) - } - } - } - entry.Chunks = chunks - } - entry.Attributes.FileSize = req.Size - file.dirtyMetadata = true - } - - if req.Valid.Mode() && entry.Attributes.FileMode != uint32(req.Mode) { - entry.Attributes.FileMode = uint32(req.Mode) - file.dirtyMetadata = true - } - - if req.Valid.Uid() && entry.Attributes.Uid != req.Uid { - entry.Attributes.Uid = req.Uid - file.dirtyMetadata = true - } - - if req.Valid.Gid() && entry.Attributes.Gid != req.Gid { - entry.Attributes.Gid = req.Gid - file.dirtyMetadata = true - } - - if req.Valid.Crtime() { - entry.Attributes.Crtime = req.Crtime.Unix() - file.dirtyMetadata = true - } - - if req.Valid.Mtime() && entry.Attributes.Mtime != req.Mtime.Unix() { - entry.Attributes.Mtime = req.Mtime.Unix() - file.dirtyMetadata = true - } - - if req.Valid.Handle() { - // fmt.Printf("file handle => %d\n", req.Handle) - } - - if file.isOpen > 0 { - return nil - } - - if !file.dirtyMetadata { - return nil - } - - return file.saveEntry(entry) - -} - -func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error { - - glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - if err := setxattr(entry, req); err != nil { - return err - } - file.dirtyMetadata = true - - if file.isOpen > 0 { - return nil - } - - return file.saveEntry(entry) - -} - -func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error { - - glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - if err := removexattr(entry, req); err != nil { - return err - } - file.dirtyMetadata = true - - if file.isOpen > 0 { - return nil - } - - return file.saveEntry(entry) - -} - -func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - - glog.V(4).Infof("file Listxattr %s", file.fullpath()) - - entry, err := file.maybeLoadEntry(ctx) - if err != nil { - return err - } - - if err := listxattr(entry, req, resp); err != nil { - return err - } - - return nil - -} - -func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { - // fsync works at OS level - // write the file chunks to the filerGrpcAddress - glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req) - - return nil -} - -func (file *File) Forget() { - t := util.NewFullPath(file.dir.FullPath(), file.Name) - glog.V(4).Infof("Forget file %s", t) - file.wfs.ReleaseHandle(t, fuse.HandleID(t.AsInode())) -} - -func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) { - - file.wfs.handlesLock.Lock() - handle, found := file.wfs.handles[file.Id()] - file.wfs.handlesLock.Unlock() - entry = file.entry - if found { - // glog.V(4).Infof("maybeLoadEntry found opened file %s/%s", file.dir.FullPath(), file.Name) - entry = handle.f.entry - } - - if entry != nil { - if len(entry.HardLinkId) == 0 { - // only always reload hard link - return entry, nil - } - } - entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name) - if err != nil { - glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err) - return entry, err - } - if entry != nil { - // file.entry = entry - } else { - glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err) - } - return entry, nil -} - -func lessThan(a, b *filer_pb.FileChunk) bool { - if a.Mtime == b.Mtime { - return a.Fid.FileKey < b.Fid.FileKey - } - return a.Mtime < b.Mtime -} - -func (file *File) addChunks(chunks []*filer_pb.FileChunk) { - - // find the earliest incoming chunk - newChunks := chunks - earliestChunk := newChunks[0] - for i := 1; i < len(newChunks); i++ { - if lessThan(earliestChunk, newChunks[i]) { - earliestChunk = newChunks[i] - } - } - - entry := file.getEntry() - if entry == nil { - return - } - - // pick out-of-order chunks from existing chunks - for _, chunk := range entry.Chunks { - if lessThan(earliestChunk, chunk) { - chunks = append(chunks, chunk) - } - } - - // sort incoming chunks - sort.Slice(chunks, func(i, j int) bool { - return lessThan(chunks[i], chunks[j]) - }) - - glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks)) - - entry.Chunks = append(entry.Chunks, newChunks...) -} - -func (file *File) saveEntry(entry *filer_pb.Entry) error { - return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - file.wfs.mapPbIdFromLocalToFiler(entry) - defer file.wfs.mapPbIdFromFilerToLocal(entry) - - request := &filer_pb.CreateEntryRequest{ - Directory: file.dir.FullPath(), - Entry: entry, - Signatures: []int32{file.wfs.signature}, - } - - glog.V(4).Infof("save file entry: %v", request) - _, err := client.CreateEntry(context.Background(), request) - if err != nil { - glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err) - return fuse.EIO - } - - file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) - - file.dirtyMetadata = false - - return nil - }) -} - -func (file *File) getEntry() *filer_pb.Entry { - return file.entry -} - -func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) { - err := file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.DownloadToLocalRequest{ - Directory: file.dir.FullPath(), - Name: entry.Name, - } - - glog.V(4).Infof("download entry: %v", request) - resp, err := client.DownloadToLocal(context.Background(), request) - if err != nil { - glog.Errorf("DownloadToLocal file %s/%s: %v", file.dir.FullPath(), file.Name, err) - return fuse.EIO - } - - entry = resp.Entry - - file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry)) - - file.dirtyMetadata = false - - return nil - }) - - return entry, err -} diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go deleted file mode 100644 index 34affddb9..000000000 --- a/weed/filesys/filehandle.go +++ /dev/null @@ -1,336 +0,0 @@ -package filesys - -import ( - "context" - "fmt" - "io" - "math" - "net/http" - "os" - "sync" - "time" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" -) - -type FileHandle struct { - // cache file has been written to - dirtyPages DirtyPages - entryViewCache []filer.VisibleInterval - reader io.ReaderAt - contentType string - handle uint64 - sync.Mutex - - f *File - RequestId fuse.RequestID // unique ID for request - NodeId fuse.NodeID // file or directory the request is about - Uid uint32 // user ID of process making request - Gid uint32 // group ID of process making request - writeOnly bool - isDeleted bool -} - -func newFileHandle(file *File, uid, gid uint32, writeOnly bool) *FileHandle { - fh := &FileHandle{ - f: file, - // dirtyPages: newContinuousDirtyPages(file, writeOnly), - dirtyPages: newTempFileDirtyPages(file, writeOnly), - Uid: uid, - Gid: gid, - } - entry := fh.f.getEntry() - if entry != nil { - entry.Attributes.FileSize = filer.FileSize(entry) - } - - return fh -} - -var _ = fs.Handle(&FileHandle{}) - -// var _ = fs.HandleReadAller(&FileHandle{}) -var _ = fs.HandleReader(&FileHandle{}) -var _ = fs.HandleFlusher(&FileHandle{}) -var _ = fs.HandleWriter(&FileHandle{}) -var _ = fs.HandleReleaser(&FileHandle{}) - -func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - - glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data)) - fh.Lock() - defer fh.Unlock() - - if req.Size <= 0 { - return nil - } - - buff := resp.Data[:cap(resp.Data)] - if req.Size > cap(resp.Data) { - // should not happen - buff = make([]byte, req.Size) - } - - totalRead, err := fh.readFromChunks(buff, req.Offset) - if err == nil || err == io.EOF { - maxStop := fh.readFromDirtyPages(buff, req.Offset) - totalRead = max(maxStop-req.Offset, totalRead) - } - - if err == io.EOF { - err = nil - } - - if err != nil { - glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err) - return fuse.EIO - } - - if totalRead > int64(len(buff)) { - glog.Warningf("%s FileHandle Read %d: [%d,%d) size %d totalRead %d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, totalRead) - totalRead = min(int64(len(buff)), totalRead) - } - if err == nil { - resp.Data = buff[:totalRead] - } - - return err -} - -func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) { - maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset) - return -} - -func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { - - entry := fh.f.getEntry() - if entry == nil { - return 0, io.EOF - } - - if entry.IsInRemoteOnly() { - glog.V(4).Infof("download remote entry %s", fh.f.fullpath()) - newEntry, err := fh.f.downloadRemoteEntry(entry) - if err != nil { - glog.V(1).Infof("download remote entry %s: %v", fh.f.fullpath(), err) - return 0, err - } - entry = newEntry - } - - fileSize := int64(filer.FileSize(entry)) - fileFullPath := fh.f.fullpath() - - if fileSize == 0 { - glog.V(1).Infof("empty fh %v", fileFullPath) - return 0, io.EOF - } - - if offset+int64(len(buff)) <= int64(len(entry.Content)) { - totalRead := copy(buff, entry.Content[offset:]) - glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead) - return int64(totalRead), nil - } - - var chunkResolveErr error - if fh.entryViewCache == nil { - fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.f.wfs.LookupFn(), entry.Chunks, 0, math.MaxInt64) - if chunkResolveErr != nil { - return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) - } - fh.reader = nil - } - - reader := fh.reader - if reader == nil { - chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64) - reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize) - } - fh.reader = reader - - totalRead, err := reader.ReadAt(buff, offset) - - if err != nil && err != io.EOF { - glog.Errorf("file handle read %s: %v", fileFullPath, err) - } - - // glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err) - - return int64(totalRead), err -} - -// Write to the file handle -func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { - - fh.Lock() - defer fh.Unlock() - - // write the request to volume servers - data := req.Data - if len(data) <= 512 { - // fuse message cacheable size - data = make([]byte, len(req.Data)) - copy(data, req.Data) - } - - entry := fh.f.getEntry() - if entry == nil { - return fuse.EIO - } - - entry.Content = nil - entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(entry.Attributes.FileSize))) - // glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data)) - - fh.dirtyPages.AddPage(req.Offset, data) - - resp.Size = len(data) - - if req.Offset == 0 { - // detect mime type - fh.contentType = http.DetectContentType(data) - fh.f.dirtyMetadata = true - } - - fh.f.dirtyMetadata = true - - return nil -} - -func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - - glog.V(4).Infof("Release %v fh %d open=%d", fh.f.fullpath(), fh.handle, fh.f.isOpen) - - fh.Lock() - defer fh.Unlock() - - fh.f.wfs.handlesLock.Lock() - fh.f.isOpen-- - fh.f.wfs.handlesLock.Unlock() - - if fh.f.isOpen <= 0 { - fh.f.entry = nil - fh.entryViewCache = nil - fh.reader = nil - - fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle)) - } - - if fh.f.isOpen < 0 { - glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0) - fh.f.isOpen = 0 - return nil - } - - return nil -} - -func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { - - glog.V(4).Infof("Flush %v fh %d", fh.f.fullpath(), fh.handle) - - if fh.isDeleted { - glog.V(4).Infof("Flush %v fh %d skip deleted", fh.f.fullpath(), fh.handle) - return nil - } - - fh.Lock() - defer fh.Unlock() - - if err := fh.doFlush(ctx, req.Header); err != nil { - glog.Errorf("Flush doFlush %s: %v", fh.f.Name, err) - return err - } - - glog.V(4).Infof("Flush %v fh %d success", fh.f.fullpath(), fh.handle) - return nil -} - -func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { - // flush works at fh level - // send the data to the OS - glog.V(4).Infof("doFlush %s fh %d", fh.f.fullpath(), fh.handle) - - if err := fh.dirtyPages.FlushData(); err != nil { - glog.Errorf("%v doFlush: %v", fh.f.fullpath(), err) - return fuse.EIO - } - - if !fh.f.dirtyMetadata { - return nil - } - - err := fh.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - entry := fh.f.getEntry() - if entry == nil { - return nil - } - - if entry.Attributes != nil { - entry.Attributes.Mime = fh.contentType - if entry.Attributes.Uid == 0 { - entry.Attributes.Uid = header.Uid - } - if entry.Attributes.Gid == 0 { - entry.Attributes.Gid = header.Gid - } - if entry.Attributes.Crtime == 0 { - entry.Attributes.Crtime = time.Now().Unix() - } - entry.Attributes.Mtime = time.Now().Unix() - entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask) - entry.Attributes.Collection, entry.Attributes.Replication = fh.dirtyPages.GetStorageOptions() - } - - request := &filer_pb.CreateEntryRequest{ - Directory: fh.f.dir.FullPath(), - Entry: entry, - Signatures: []int32{fh.f.wfs.signature}, - } - - glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(entry.Chunks)) - for i, chunk := range entry.Chunks { - glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) - } - - manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks) - - chunks, _ := filer.CompactFileChunks(fh.f.wfs.LookupFn(), nonManifestChunks) - chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.fullpath(), fh.dirtyPages.GetWriteOnly()), chunks) - if manifestErr != nil { - // not good, but should be ok - glog.V(0).Infof("MaybeManifestize: %v", manifestErr) - } - entry.Chunks = append(chunks, manifestChunks...) - - fh.f.wfs.mapPbIdFromLocalToFiler(request.Entry) - defer fh.f.wfs.mapPbIdFromFilerToLocal(request.Entry) - - if err := filer_pb.CreateEntry(client, request); err != nil { - glog.Errorf("fh flush create %s: %v", fh.f.fullpath(), err) - return fmt.Errorf("fh flush create %s: %v", fh.f.fullpath(), err) - } - - fh.f.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) - - return nil - }) - - if err == nil { - fh.f.dirtyMetadata = false - } - - if err != nil { - glog.Errorf("%v fh %d flush: %v", fh.f.fullpath(), fh.handle, err) - return fuse.EIO - } - - return nil -} diff --git a/weed/filesys/fscache.go b/weed/filesys/fscache.go deleted file mode 100644 index 6b1012090..000000000 --- a/weed/filesys/fscache.go +++ /dev/null @@ -1,213 +0,0 @@ -package filesys - -import ( - "sync" - - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/util" -) - -type FsCache struct { - root *FsNode - sync.RWMutex -} -type FsNode struct { - parent *FsNode - node fs.Node - name string - childrenLock sync.RWMutex - children map[string]*FsNode -} - -func newFsCache(root fs.Node) *FsCache { - return &FsCache{ - root: &FsNode{ - node: root, - }, - } -} - -func (c *FsCache) GetFsNode(path util.FullPath) fs.Node { - - c.RLock() - defer c.RUnlock() - - return c.doGetFsNode(path) -} - -func (c *FsCache) doGetFsNode(path util.FullPath) fs.Node { - t := c.root - for _, p := range path.Split() { - t = t.findChild(p) - if t == nil { - return nil - } - } - return t.node -} - -func (c *FsCache) SetFsNode(path util.FullPath, node fs.Node) { - - c.Lock() - defer c.Unlock() - - c.doSetFsNode(path, node) -} - -func (c *FsCache) doSetFsNode(path util.FullPath, node fs.Node) { - t := c.root - for _, p := range path.Split() { - t = t.ensureChild(p) - } - t.node = node -} - -func (c *FsCache) EnsureFsNode(path util.FullPath, genNodeFn func() fs.Node) fs.Node { - - c.Lock() - defer c.Unlock() - - t := c.doGetFsNode(path) - if t != nil { - return t - } - t = genNodeFn() - c.doSetFsNode(path, t) - return t -} - -func (c *FsCache) DeleteFsNode(path util.FullPath) { - - c.Lock() - defer c.Unlock() - - t := c.root - for _, p := range path.Split() { - t = t.findChild(p) - if t == nil { - return - } - } - if t.parent != nil { - t.parent.disconnectChild(t) - } - t.deleteSelf() -} - -// oldPath and newPath are full path including the new name -func (c *FsCache) Move(oldPath util.FullPath, newPath util.FullPath) *FsNode { - - c.Lock() - defer c.Unlock() - - // find old node - src := c.root - for _, p := range oldPath.Split() { - src = src.findChild(p) - if src == nil { - return src - } - } - if src.parent != nil { - src.parent.disconnectChild(src) - } - - // find new node - target := c.root - for _, p := range newPath.Split() { - target = target.ensureChild(p) - } - parent := target.parent - if dir, ok := src.node.(*Dir); ok { - dir.name = target.name // target is not Dir, but a shortcut - } - if f, ok := src.node.(*File); ok { - f.Name = target.name - entry := f.getEntry() - if entry != nil { - entry.Name = f.Name - } - } - parent.disconnectChild(target) - - target.deleteSelf() - - src.name = target.name - src.connectToParent(parent) - - return src -} - -func (n *FsNode) connectToParent(parent *FsNode) { - n.parent = parent - oldNode := parent.findChild(n.name) - if oldNode != nil { - oldNode.deleteSelf() - } - if dir, ok := n.node.(*Dir); ok { - if parent.node != nil { - dir.parent = parent.node.(*Dir) - } - } - if f, ok := n.node.(*File); ok { - if parent.node != nil { - f.dir = parent.node.(*Dir) - } - } - n.childrenLock.Lock() - parent.children[n.name] = n - n.childrenLock.Unlock() -} - -func (n *FsNode) findChild(name string) *FsNode { - n.childrenLock.RLock() - defer n.childrenLock.RUnlock() - - child, found := n.children[name] - if found { - return child - } - return nil -} - -func (n *FsNode) ensureChild(name string) *FsNode { - n.childrenLock.Lock() - defer n.childrenLock.Unlock() - - if n.children == nil { - n.children = make(map[string]*FsNode) - } - child, found := n.children[name] - if found { - return child - } - t := &FsNode{ - parent: n, - node: nil, - name: name, - children: nil, - } - n.children[name] = t - return t -} - -func (n *FsNode) disconnectChild(child *FsNode) { - n.childrenLock.Lock() - delete(n.children, child.name) - n.childrenLock.Unlock() - child.parent = nil -} - -func (n *FsNode) deleteSelf() { - n.childrenLock.Lock() - for _, child := range n.children { - child.deleteSelf() - } - n.children = nil - n.childrenLock.Unlock() - - n.node = nil - n.parent = nil - -} diff --git a/weed/filesys/fscache_test.go b/weed/filesys/fscache_test.go deleted file mode 100644 index 1152eb32e..000000000 --- a/weed/filesys/fscache_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package filesys - -import ( - "testing" - - "github.com/chrislusf/seaweedfs/weed/util" -) - -func TestPathSplit(t *testing.T) { - parts := util.FullPath("/").Split() - if len(parts) != 0 { - t.Errorf("expecting an empty list, but getting %d", len(parts)) - } - - parts = util.FullPath("/readme.md").Split() - if len(parts) != 1 { - t.Errorf("expecting an empty list, but getting %d", len(parts)) - } - -} - -func TestFsCache(t *testing.T) { - - cache := newFsCache(nil) - - x := cache.GetFsNode(util.FullPath("/y/x")) - if x != nil { - t.Errorf("wrong node!") - } - - p := util.FullPath("/a/b/c") - cache.SetFsNode(p, &File{Name: "cc"}) - tNode := cache.GetFsNode(p) - tFile := tNode.(*File) - if tFile.Name != "cc" { - t.Errorf("expecting a FsNode") - } - - cache.SetFsNode(util.FullPath("/a/b/d"), &File{Name: "dd"}) - cache.SetFsNode(util.FullPath("/a/b/e"), &File{Name: "ee"}) - cache.SetFsNode(util.FullPath("/a/b/f"), &File{Name: "ff"}) - cache.SetFsNode(util.FullPath("/z"), &File{Name: "zz"}) - cache.SetFsNode(util.FullPath("/a"), &File{Name: "aa"}) - - b := cache.GetFsNode(util.FullPath("/a/b")) - if b != nil { - t.Errorf("unexpected node!") - } - - a := cache.GetFsNode(util.FullPath("/a")) - if a == nil { - t.Errorf("missing node!") - } - - cache.DeleteFsNode(util.FullPath("/a")) - if b != nil { - t.Errorf("unexpected node!") - } - - a = cache.GetFsNode(util.FullPath("/a")) - if a != nil { - t.Errorf("wrong DeleteFsNode!") - } - - z := cache.GetFsNode(util.FullPath("/z")) - if z == nil { - t.Errorf("missing node!") - } - - y := cache.GetFsNode(util.FullPath("/x/y")) - if y != nil { - t.Errorf("wrong node!") - } - -} - -func TestFsCacheMove(t *testing.T) { - - cache := newFsCache(nil) - - cache.SetFsNode(util.FullPath("/a/b/d"), &File{Name: "dd"}) - cache.SetFsNode(util.FullPath("/a/b/e"), &File{Name: "ee"}) - cache.SetFsNode(util.FullPath("/z"), &File{Name: "zz"}) - cache.SetFsNode(util.FullPath("/a"), &File{Name: "aa"}) - - cache.Move(util.FullPath("/a/b"), util.FullPath("/z/x")) - - d := cache.GetFsNode(util.FullPath("/z/x/d")) - if d == nil { - t.Errorf("unexpected nil node!") - } - if d.(*File).Name != "dd" { - t.Errorf("unexpected non dd node!") - } - -} - -func TestFsCacheMove2(t *testing.T) { - - cache := newFsCache(nil) - - cache.SetFsNode(util.FullPath("/a/b/d"), &File{Name: "dd"}) - cache.SetFsNode(util.FullPath("/a/b/e"), &File{Name: "ee"}) - - cache.Move(util.FullPath("/a/b/d"), util.FullPath("/a/b/e")) - - d := cache.GetFsNode(util.FullPath("/a/b/e")) - if d == nil { - t.Errorf("unexpected nil node!") - } - if d.(*File).Name != "e" { - t.Errorf("unexpected node!") - } - -} diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go deleted file mode 100644 index 07098bf6b..000000000 --- a/weed/filesys/meta_cache/meta_cache_init.go +++ /dev/null @@ -1,47 +0,0 @@ -package meta_cache - -import ( - "context" - "fmt" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" -) - -func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath) error { - - return mc.visitedBoundary.EnsureVisited(dirPath, func(path util.FullPath) (childDirectories []string, err error) { - - glog.V(4).Infof("ReadDirAllEntries %s ...", path) - - util.Retry("ReadDirAllEntries", func() error { - err = filer_pb.ReadDirAllEntries(client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error { - entry := filer.FromPbEntry(string(path), pbEntry) - if IsHiddenSystemEntry(string(path), entry.Name()) { - return nil - } - if err := mc.doInsertEntry(context.Background(), entry); err != nil { - glog.V(0).Infof("read %s: %v", entry.FullPath, err) - return err - } - if entry.IsDirectory() { - childDirectories = append(childDirectories, entry.Name()) - } - return nil - }) - return err - }) - - if err != nil { - err = fmt.Errorf("list %s: %v", path, err) - } - - return - }) -} - -func IsHiddenSystemEntry(dir, name string) bool { - return dir == "/" && (name == "topics" || name == "etc") -} diff --git a/weed/filesys/unimplemented.go b/weed/filesys/unimplemented.go deleted file mode 100644 index 5c2dcf0e1..000000000 --- a/weed/filesys/unimplemented.go +++ /dev/null @@ -1,22 +0,0 @@ -package filesys - -import ( - "context" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" -) - -// https://github.com/bazil/fuse/issues/130 - -var _ = fs.NodeAccesser(&Dir{}) - -func (dir *Dir) Access(ctx context.Context, req *fuse.AccessRequest) error { - return fuse.ENOSYS -} - -var _ = fs.NodeAccesser(&File{}) - -func (file *File) Access(ctx context.Context, req *fuse.AccessRequest) error { - return fuse.ENOSYS -} diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go deleted file mode 100644 index 84d4bdfa2..000000000 --- a/weed/filesys/wfs.go +++ /dev/null @@ -1,303 +0,0 @@ -package filesys - -import ( - "context" - "fmt" - "math" - "math/rand" - "os" - "path" - "path/filepath" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "github.com/chrislusf/seaweedfs/weed/wdclient" - - "google.golang.org/grpc" - - "github.com/chrislusf/seaweedfs/weed/util/grace" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" -) - -type Option struct { - MountDirectory string - FilerAddresses []string - filerIndex int - FilerGrpcAddresses []string - GrpcDialOption grpc.DialOption - FilerMountRootPath string - Collection string - Replication string - TtlSec int32 - DiskType types.DiskType - ChunkSizeLimit int64 - ConcurrentWriters int - CacheDir string - CacheSizeMB int64 - DataCenter string - Umask os.FileMode - - MountUid uint32 - MountGid uint32 - MountMode os.FileMode - MountCtime time.Time - MountMtime time.Time - MountParentInode uint64 - - VolumeServerAccess string // how to access volume servers - Cipher bool // whether encrypt data on volume server - UidGidMapper *meta_cache.UidGidMapper - - uniqueCacheDir string - uniqueCacheTempPageDir string -} - -var _ = fs.FS(&WFS{}) -var _ = fs.FSStatfser(&WFS{}) - -type WFS struct { - option *Option - - // contains all open handles, protected by handlesLock - handlesLock sync.Mutex - handles map[uint64]*FileHandle - - bufPool sync.Pool - - stats statsCache - - root fs.Node - fsNodeCache *FsCache - - chunkCache *chunk_cache.TieredChunkCache - metaCache *meta_cache.MetaCache - signature int32 - - // throttle writers - concurrentWriters *util.LimitedConcurrentExecutor - Server *fs.Server -} -type statsCache struct { - filer_pb.StatisticsResponse - lastChecked int64 // unix time in seconds -} - -func NewSeaweedFileSystem(option *Option) *WFS { - wfs := &WFS{ - option: option, - handles: make(map[uint64]*FileHandle), - bufPool: sync.Pool{ - New: func() interface{} { - return make([]byte, option.ChunkSizeLimit) - }, - }, - signature: util.RandomInt32(), - } - wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses)) - wfs.option.setupUniqueCacheDirectory() - if option.CacheSizeMB > 0 { - wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024) - } - - wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), util.FullPath(option.FilerMountRootPath), option.UidGidMapper, func(filePath util.FullPath) { - - fsNode := NodeWithId(filePath.AsInode()) - if err := wfs.Server.InvalidateNodeData(fsNode); err != nil { - glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err) - } - - dir, name := filePath.DirAndName() - parent := NodeWithId(util.FullPath(dir).AsInode()) - if dir == option.FilerMountRootPath { - parent = NodeWithId(1) - } - if err := wfs.Server.InvalidateEntry(parent, name); err != nil { - glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err) - } - }) - grace.OnInterrupt(func() { - wfs.metaCache.Shutdown() - }) - - wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs, id: 1} - wfs.fsNodeCache = newFsCache(wfs.root) - - if wfs.option.ConcurrentWriters > 0 { - wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters) - } - - return wfs -} - -func (wfs *WFS) StartBackgroundTasks() { - startTime := time.Now() - go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano()) -} - -func (wfs *WFS) Root() (fs.Node, error) { - return wfs.root, nil -} - -func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32, writeOnly bool) (fileHandle *FileHandle) { - - fullpath := file.fullpath() - glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid) - - inodeId := file.Id() - - wfs.handlesLock.Lock() - existingHandle, found := wfs.handles[inodeId] - if found && existingHandle != nil && existingHandle.f.isOpen > 0 { - existingHandle.f.isOpen++ - wfs.handlesLock.Unlock() - existingHandle.dirtyPages.SetWriteOnly(writeOnly) - glog.V(4).Infof("Reuse AcquiredHandle %s open %d", fullpath, existingHandle.f.isOpen) - return existingHandle - } - wfs.handlesLock.Unlock() - - entry, _ := file.maybeLoadEntry(context.Background()) - file.entry = entry - fileHandle = newFileHandle(file, uid, gid, writeOnly) - - wfs.handlesLock.Lock() - file.isOpen++ - wfs.handles[inodeId] = fileHandle - wfs.handlesLock.Unlock() - fileHandle.handle = inodeId - - glog.V(4).Infof("Acquired new Handle %s open %d", fullpath, file.isOpen) - return -} - -func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) { - wfs.handlesLock.Lock() - defer wfs.handlesLock.Unlock() - - glog.V(4).Infof("ReleaseHandle %s id %d current handles length %d", fullpath, handleId, len(wfs.handles)) - - delete(wfs.handles, uint64(handleId)) - - return -} - -// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser -func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error { - - glog.V(4).Infof("reading fs stats: %+v", req) - - if wfs.stats.lastChecked < time.Now().Unix()-20 { - - err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.StatisticsRequest{ - Collection: wfs.option.Collection, - Replication: wfs.option.Replication, - Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), - DiskType: string(wfs.option.DiskType), - } - - glog.V(4).Infof("reading filer stats: %+v", request) - resp, err := client.Statistics(context.Background(), request) - if err != nil { - glog.V(0).Infof("reading filer stats %v: %v", request, err) - return err - } - glog.V(4).Infof("read filer stats: %+v", resp) - - wfs.stats.TotalSize = resp.TotalSize - wfs.stats.UsedSize = resp.UsedSize - wfs.stats.FileCount = resp.FileCount - wfs.stats.lastChecked = time.Now().Unix() - - return nil - }) - if err != nil { - glog.V(0).Infof("filer Statistics: %v", err) - return err - } - } - - totalDiskSize := wfs.stats.TotalSize - usedDiskSize := wfs.stats.UsedSize - actualFileCount := wfs.stats.FileCount - - // Compute the total number of available blocks - resp.Blocks = totalDiskSize / blockSize - - // Compute the number of used blocks - numBlocks := uint64(usedDiskSize / blockSize) - - // Report the number of free and available blocks for the block size - resp.Bfree = resp.Blocks - numBlocks - resp.Bavail = resp.Blocks - numBlocks - resp.Bsize = uint32(blockSize) - - // Report the total number of possible files in the file system (and those free) - resp.Files = math.MaxInt64 - resp.Ffree = math.MaxInt64 - actualFileCount - - // Report the maximum length of a name and the minimum fragment size - resp.Namelen = 1024 - resp.Frsize = uint32(blockSize) - - return nil -} - -func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) { - if entry.Attributes == nil { - return - } - entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) -} -func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { - if entry.Attributes == nil { - return - } - entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) -} - -func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { - if wfs.option.VolumeServerAccess == "filerProxy" { - return func(fileId string) (targetUrls []string, err error) { - return []string{"http://" + wfs.getCurrentFiler() + "/?proxyChunkId=" + fileId}, nil - } - } - return filer.LookupFn(wfs) -} -func (wfs *WFS) getCurrentFiler() string { - return wfs.option.FilerAddresses[wfs.option.filerIndex] -} - -func (option *Option) setupUniqueCacheDirectory() { - cacheUniqueId := util.Md5String([]byte(option.MountDirectory + option.FilerGrpcAddresses[0] + option.FilerMountRootPath + util.Version()))[0:8] - option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId) - option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "sw") - os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask) -} - -func (option *Option) getTempFilePageDir() string { - return option.uniqueCacheTempPageDir -} -func (option *Option) getUniqueCacheDir() string { - return option.uniqueCacheDir -} - -type NodeWithId uint64 - -func (n NodeWithId) Id() uint64 { - return uint64(n) -} -func (n NodeWithId) Attr(ctx context.Context, attr *fuse.Attr) error { - return nil -} diff --git a/weed/filesys/xattr.go b/weed/filesys/xattr.go deleted file mode 100644 index 4b2ee0064..000000000 --- a/weed/filesys/xattr.go +++ /dev/null @@ -1,138 +0,0 @@ -package filesys - -import ( - "context" - - "github.com/seaweedfs/fuse" - - "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" -) - -func getxattr(entry *filer_pb.Entry, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - - if entry == nil { - return fuse.ErrNoXattr - } - if entry.Extended == nil { - return fuse.ErrNoXattr - } - data, found := entry.Extended[req.Name] - if !found { - return fuse.ErrNoXattr - } - if req.Position < uint32(len(data)) { - size := req.Size - if req.Position+size >= uint32(len(data)) { - size = uint32(len(data)) - req.Position - } - if size == 0 { - resp.Xattr = data[req.Position:] - } else { - resp.Xattr = data[req.Position : req.Position+size] - } - } - - return nil - -} - -func setxattr(entry *filer_pb.Entry, req *fuse.SetxattrRequest) error { - - if entry == nil { - return fuse.EIO - } - - if entry.Extended == nil { - entry.Extended = make(map[string][]byte) - } - data, _ := entry.Extended[req.Name] - - newData := make([]byte, int(req.Position)+len(req.Xattr)) - - copy(newData, data) - - copy(newData[int(req.Position):], req.Xattr) - - entry.Extended[req.Name] = newData - - return nil - -} - -func removexattr(entry *filer_pb.Entry, req *fuse.RemovexattrRequest) error { - - if entry == nil { - return fuse.ErrNoXattr - } - - if entry.Extended == nil { - return fuse.ErrNoXattr - } - - _, found := entry.Extended[req.Name] - - if !found { - return fuse.ErrNoXattr - } - - delete(entry.Extended, req.Name) - - return nil - -} - -func listxattr(entry *filer_pb.Entry, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - - if entry == nil { - return fuse.EIO - } - - for k := range entry.Extended { - resp.Append(k) - } - - size := req.Size - if req.Position+size >= uint32(len(resp.Xattr)) { - size = uint32(len(resp.Xattr)) - req.Position - } - - if size == 0 { - resp.Xattr = resp.Xattr[req.Position:] - } else { - resp.Xattr = resp.Xattr[req.Position : req.Position+size] - } - - return nil - -} - -func (wfs *WFS) maybeLoadEntry(dir, name string) (entry *filer_pb.Entry, err error) { - - fullpath := util.NewFullPath(dir, name) - // glog.V(3).Infof("read entry cache miss %s", fullpath) - - // return a valid entry for the mount root - if string(fullpath) == wfs.option.FilerMountRootPath { - return &filer_pb.Entry{ - Name: name, - IsDirectory: true, - Attributes: &filer_pb.FuseAttributes{ - Mtime: wfs.option.MountMtime.Unix(), - FileMode: uint32(wfs.option.MountMode), - Uid: wfs.option.MountUid, - Gid: wfs.option.MountGid, - Crtime: wfs.option.MountCtime.Unix(), - }, - }, nil - } - - // read from async meta cache - meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir)) - cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath) - if cacheErr == filer_pb.ErrNotFound { - return nil, fuse.ENOENT - } - return cachedEntry.ToProtoEntry(), cacheErr -} diff --git a/weed/ftpd/ftp_server.go b/weed/ftpd/ftp_server.go index 4a0dca2c3..253ff3edd 100644 --- a/weed/ftpd/ftp_server.go +++ b/weed/ftpd/ftp_server.go @@ -3,7 +3,7 @@ package ftpd import ( "crypto/tls" "errors" - "fmt" + "github.com/chrislusf/seaweedfs/weed/util" "net" ftpserver "github.com/fclairamb/ftpserverlib" @@ -51,7 +51,7 @@ func (s *SftpServer) GetSettings() (*ftpserver.Settings, error) { return &ftpserver.Settings{ Listener: s.ftpListener, - ListenAddr: fmt.Sprintf("%s:%d", s.option.IpBind, s.option.Port), + ListenAddr: util.JoinHostPort(s.option.IpBind, s.option.Port), PublicHost: s.option.IP, PassiveTransferPortRange: portRange, ActiveTransferPortNon20: true, diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go index 7765d9e95..a59834e88 100644 --- a/weed/iamapi/iamapi_handlers.go +++ b/weed/iamapi/iamapi_handlers.go @@ -8,7 +8,7 @@ import ( "net/http" ) -func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) { +func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, err error, object string, value string, msg error) { errCode := err.Error() errorResp := ErrorResponse{} errorResp.Error.Type = "Sender" @@ -22,10 +22,10 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu case iam.ErrCodeNoSuchEntityException: msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value) errorResp.Error.Message = &msg - s3err.WriteXMLResponse(w, http.StatusNotFound, errorResp) + s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp) case iam.ErrCodeServiceFailureException: - s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp) + s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp) default: - s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp) + s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp) } } diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 0826ce336..488e92aa5 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -4,10 +4,6 @@ import ( "crypto/sha1" "encoding/json" "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "math/rand" "net/http" "net/url" @@ -16,6 +12,11 @@ import ( "sync" "time" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/aws/aws-sdk-go/service/iam" ) @@ -155,6 +156,22 @@ func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName str return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException) } +func (iama *IamApiServer) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp UpdateUserResponse, err error) { + userName := values.Get("UserName") + newUserName := values.Get("NewUserName") + if newUserName != "" { + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Name = newUserName + return resp, nil + } + } + } else { + return resp, nil + } + return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException) +} + func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) { if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil { return PolicyDocument{}, err @@ -360,9 +377,38 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu return resp } +// handleImplicitUsername adds username who signs the request to values if 'username' is not specified +// According to https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/create-access-key.html/ +// "If you do not specify a user name, IAM determines the user name implicitly based on the Amazon Web +// Services access key ID signing the request." +func handleImplicitUsername(r *http.Request, values url.Values) { + if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" { + return + } + // get username who signs the request. For a typical Authorization: + // "AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type; + // host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8", + // the "test1" will be extracted as the username + glog.V(4).Infof("Authorization field: %v", r.Header["Authorization"][0]) + s := strings.Split(r.Header["Authorization"][0], "Credential=") + if len(s) < 2 { + return + } + s = strings.Split(s[1], ",") + if len(s) < 2 { + return + } + s = strings.Split(s[0], "/") + if len(s) < 5 { + return + } + userName := s[2] + values.Set("UserName", userName) +} + func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return } values := r.PostForm @@ -370,7 +416,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { s3cfgLock.RLock() s3cfg := &iam_pb.S3ApiConfiguration{} if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } s3cfgLock.RUnlock() @@ -384,6 +430,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { response = iama.ListUsers(s3cfg, values) changed = false case "ListAccessKeys": + handleImplicitUsername(r, values) response = iama.ListAccessKeys(s3cfg, values) changed = false case "CreateUser": @@ -392,52 +439,62 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { userName := values.Get("UserName") response, err = iama.GetUser(s3cfg, userName) if err != nil { - writeIamErrorResponse(w, err, "user", userName, nil) + writeIamErrorResponse(w, r, err, "user", userName, nil) return } changed = false + case "UpdateUser": + response, err = iama.UpdateUser(s3cfg, values) + if err != nil { + glog.Errorf("UpdateUser: %+v", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } case "DeleteUser": userName := values.Get("UserName") response, err = iama.DeleteUser(s3cfg, userName) if err != nil { - writeIamErrorResponse(w, err, "user", userName, nil) + writeIamErrorResponse(w, r, err, "user", userName, nil) return } case "CreateAccessKey": + handleImplicitUsername(r, values) response = iama.CreateAccessKey(s3cfg, values) case "DeleteAccessKey": + handleImplicitUsername(r, values) response = iama.DeleteAccessKey(s3cfg, values) case "CreatePolicy": response, err = iama.CreatePolicy(s3cfg, values) if err != nil { glog.Errorf("CreatePolicy: %+v", err) - s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return } case "PutUserPolicy": response, err = iama.PutUserPolicy(s3cfg, values) if err != nil { glog.Errorf("PutUserPolicy: %+v", err) - s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return } case "GetUserPolicy": response, err = iama.GetUserPolicy(s3cfg, values) if err != nil { - writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil) + writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil) return } changed = false case "DeleteUserPolicy": if response, err = iama.DeleteUserPolicy(s3cfg, values); err != nil { - writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil) + writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil) + return } default: errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented) errorResponse := ErrorResponse{} errorResponse.Error.Code = &errNotImplemented.Code errorResponse.Error.Message = &errNotImplemented.Description - s3err.WriteXMLResponse(w, errNotImplemented.HTTPStatusCode, errorResponse) + s3err.WriteXMLResponse(w, r, errNotImplemented.HTTPStatusCode, errorResponse) return } if changed { @@ -445,9 +502,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg) s3cfgLock.Unlock() if err != nil { - writeIamErrorResponse(w, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err) + writeIamErrorResponse(w, r, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err) return } } - s3err.WriteXMLResponse(w, http.StatusOK, response) + s3err.WriteXMLResponse(w, r, http.StatusOK, response) } diff --git a/weed/iamapi/iamapi_response.go b/weed/iamapi/iamapi_response.go index 77328b608..df9443f0d 100644 --- a/weed/iamapi/iamapi_response.go +++ b/weed/iamapi/iamapi_response.go @@ -66,6 +66,11 @@ type GetUserResponse struct { } `xml:"GetUserResult"` } +type UpdateUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateUserResponse"` +} + type CreateAccessKeyResponse struct { CommonResponse XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"` diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index ec718bd41..62c7f867c 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -18,7 +18,6 @@ import ( "github.com/gorilla/mux" "google.golang.org/grpc" "net/http" - "strings" ) type IamS3ApiConfig interface { @@ -34,11 +33,10 @@ type IamS3ApiConfigure struct { } type IamServerOption struct { - Masters string - Filer string - Port int - FilerGrpcAddress string - GrpcDialOption grpc.DialOption + Masters map[string]pb.ServerAddress + Filer pb.ServerAddress + Port int + GrpcDialOption grpc.DialOption } type IamApiServer struct { @@ -51,7 +49,7 @@ var s3ApiConfigure IamS3ApiConfig func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) { s3ApiConfigure = IamS3ApiConfigure{ option: option, - masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(option.Masters, ",")), + masterClient: wdclient.NewMasterClient(option.GrpcDialOption, "", "iam", "", "", option.Masters), } s3Option := s3api.S3ApiServerOption{Filer: option.Filer} iamApiServer = &IamApiServer{ @@ -78,7 +76,7 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) { func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { var buf bytes.Buffer - err = pb.WithGrpcFilerClient(iam.option.FilerGrpcAddress, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf); err != nil { return err } @@ -100,24 +98,20 @@ func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigurat if err := filer.ProtoToText(&buf, s3cfg); err != nil { return fmt.Errorf("ProtoToText: %s", err) } - return pb.WithGrpcFilerClient( - iam.option.FilerGrpcAddress, - iam.option.GrpcDialOption, - func(client filer_pb.SeaweedFilerClient) error { - err = util.Retry("saveIamIdentity", func() error { - return filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()) - }) - if err != nil { - return err - } - return nil - }, - ) + return pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = util.Retry("saveIamIdentity", func() error { + return filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()) + }) + if err != nil { + return err + } + return nil + }) } func (iam IamS3ApiConfigure) GetPolicies(policies *Policies) (err error) { var buf bytes.Buffer - err = pb.WithGrpcFilerClient(iam.option.FilerGrpcAddress, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirecotry, filer.IamPoliciesFile, &buf); err != nil { return err } @@ -141,14 +135,10 @@ func (iam IamS3ApiConfigure) PutPolicies(policies *Policies) (err error) { if b, err = json.Marshal(policies); err != nil { return err } - return pb.WithGrpcFilerClient( - iam.option.FilerGrpcAddress, - iam.option.GrpcDialOption, - func(client filer_pb.SeaweedFilerClient) error { - if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamPoliciesFile, b); err != nil { - return err - } - return nil - }, - ) + return pb.WithGrpcFilerClient(false, iam.option.Filer, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamPoliciesFile, b); err != nil { + return err + } + return nil + }) } diff --git a/weed/iamapi/iamapi_test.go b/weed/iamapi/iamapi_test.go index 09aaf0ac8..375e9a2f3 100644 --- a/weed/iamapi/iamapi_test.go +++ b/weed/iamapi/iamapi_test.go @@ -2,6 +2,11 @@ package iamapi import ( "encoding/xml" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/iam" @@ -9,9 +14,6 @@ import ( "github.com/gorilla/mux" "github.com/jinzhu/copier" "github.com/stretchr/testify/assert" - "net/http" - "net/http/httptest" - "testing" ) var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error) @@ -161,8 +163,20 @@ func TestGetUserPolicy(t *testing.T) { assert.Equal(t, http.StatusOK, response.Code) } -func TestDeleteUser(t *testing.T) { +func TestUpdateUser(t *testing.T) { userName := aws.String("Test") + newUserName := aws.String("Test-New") + params := &iam.UpdateUserInput{NewUserName: newUserName, UserName: userName} + req, _ := iam.New(session.New()).UpdateUserRequest(params) + _ = req.Build() + out := UpdateUserResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +func TestDeleteUser(t *testing.T) { + userName := aws.String("Test-New") params := &iam.DeleteUserInput{UserName: userName} req, _ := iam.New(session.New()).DeleteUserRequest(params) _ = req.Build() @@ -179,3 +193,24 @@ func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorde apiRouter.ServeHTTP(rr, req) return rr, xml.Unmarshal(rr.Body.Bytes(), &v) } + +func TestHandleImplicitUsername(t *testing.T) { + var tests = []struct { + r *http.Request + values url.Values + userName string + }{ + {&http.Request{}, url.Values{}, ""}, + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, "test1"}, + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request SignedHeaders=content-type;host;x-amz-date Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, + } + + for i, test := range tests { + handleImplicitUsername(test.r, test.values) + if un := test.values.Get("UserName"); un != test.userName { + t.Errorf("No.%d: Got: %v, Expected: %v", i, un, test.userName) + } + } +} diff --git a/weed/images/orientation_test.go b/weed/images/orientation_test.go index 32fa38f76..92bf4fb8f 100644 --- a/weed/images/orientation_test.go +++ b/weed/images/orientation_test.go @@ -1,7 +1,7 @@ package images import ( - "io/ioutil" + "github.com/chrislusf/seaweedfs/weed/util" "os" "testing" ) @@ -9,11 +9,11 @@ import ( func TestXYZ(t *testing.T) { fname := "sample1.jpg" - dat, _ := ioutil.ReadFile(fname) + dat, _ := os.ReadFile(fname) fixed_data := FixJpgOrientation(dat) - ioutil.WriteFile("fixed1.jpg", fixed_data, 0644) + util.WriteFile("fixed1.jpg", fixed_data, 0644) os.Remove("fixed1.jpg") diff --git a/weed/images/resizing_test.go b/weed/images/resizing_test.go index d12f799d8..035c42b4d 100644 --- a/weed/images/resizing_test.go +++ b/weed/images/resizing_test.go @@ -2,7 +2,7 @@ package images import ( "bytes" - "io/ioutil" + "github.com/chrislusf/seaweedfs/weed/util" "os" "testing" ) @@ -10,13 +10,13 @@ import ( func TestResizing(t *testing.T) { fname := "sample2.webp" - dat, _ := ioutil.ReadFile(fname) + dat, _ := os.ReadFile(fname) resized, _, _ := Resized(".webp", bytes.NewReader(dat), 100, 30, "") buf := new(bytes.Buffer) buf.ReadFrom(resized) - ioutil.WriteFile("resized1.png", buf.Bytes(), 0644) + util.WriteFile("resized1.png", buf.Bytes(), 0644) os.Remove("resized1.png") diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index 40c807164..9a31a8ac0 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -24,7 +24,7 @@ func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messag dir, name := util.FullPath(targetFile).DirAndName() // append the chunk - if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := broker.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AppendToEntryRequest{ Directory: dir, @@ -51,7 +51,7 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf var assignResult = &operation.AssignResult{} // assign a volume location - if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := broker.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { assignErr := util.Retry("assignVolume", func() error { request := &filer_pb.AssignVolumeRequest{ @@ -71,8 +71,9 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf assignResult.Auth = security.EncodedJwt(resp.Auth) assignResult.Fid = resp.FileId - assignResult.Url = resp.Url - assignResult.PublicUrl = resp.PublicUrl + assignResult.Url = resp.Location.Url + assignResult.PublicUrl = resp.Location.PublicUrl + assignResult.GrpcPort = int(resp.Location.GrpcPort) assignResult.Count = uint64(resp.Count) return nil @@ -88,7 +89,16 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf // upload data targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) - uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth) + uploadOption := &operation.UploadOption{ + UploadUrl: targetUrl, + Filename: "", + Cipher: broker.option.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: assignResult.Auth, + } + uploadResult, err := operation.UploadData(data, uploadOption) if err != nil { return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) } @@ -98,10 +108,10 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf var _ = filer_pb.FilerClient(&MessageBroker{}) -func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err error) { +func (broker *MessageBroker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) { for _, filer := range broker.option.Filers { - if err = pb.WithFilerClient(filer, broker.grpcDialOption, fn); err != nil { + if err = pb.WithFilerClient(streamingMode, filer, broker.grpcDialOption, fn); err != nil { if err == io.EOF { return } diff --git a/weed/messaging/broker/broker_grpc_server_discovery.go b/weed/messaging/broker/broker_grpc_server_discovery.go index 3c14f3220..5cd8edd33 100644 --- a/weed/messaging/broker/broker_grpc_server_discovery.go +++ b/weed/messaging/broker/broker_grpc_server_discovery.go @@ -3,6 +3,8 @@ package broker import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" "time" "github.com/chrislusf/seaweedfs/weed/glog" @@ -32,7 +34,7 @@ func (broker *MessageBroker) FindBroker(c context.Context, request *messaging_pb targetTopicPartition := fmt.Sprintf(TopicPartitionFmt, request.Namespace, request.Topic, request.Parition) for _, filer := range broker.option.Filers { - err := broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + err := broker.withFilerClient(false, filer, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.LocateBroker(context.Background(), &filer_pb.LocateBrokerRequest{ Resource: targetTopicPartition, }) @@ -62,16 +64,18 @@ func (broker *MessageBroker) FindBroker(c context.Context, request *messaging_pb func (broker *MessageBroker) checkFilers() { // contact a filer about masters - var masters []string + var masters []pb.ServerAddress found := false for !found { for _, filer := range broker.option.Filers { - err := broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + err := broker.withFilerClient(false, filer, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return err } - masters = append(masters, resp.Masters...) + for _, m := range resp.Masters { + masters = append(masters, pb.ServerAddress(m)) + } return nil }) if err == nil { @@ -85,19 +89,21 @@ func (broker *MessageBroker) checkFilers() { glog.V(0).Infof("received master list: %s", masters) // contact each masters for filers - var filers []string + var filers []pb.ServerAddress found = false for !found { for _, master := range masters { - err := broker.withMasterClient(master, func(client master_pb.SeaweedClient) error { - resp, err := client.ListMasterClients(context.Background(), &master_pb.ListMasterClientsRequest{ - ClientType: "filer", + err := broker.withMasterClient(false, master, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, }) if err != nil { return err } - filers = append(filers, resp.GrpcAddresses...) + for _, clusterNode := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(clusterNode.Address)) + } return nil }) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index d21fb351f..20d529239 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -2,6 +2,7 @@ package broker import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/log_buffer" "io" "strings" @@ -116,7 +117,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lastReadTime = time.Unix(0, processedTsNs) } - lastReadTime, err = lock.logBuffer.LoopProcessLogData("broker", lastReadTime, func() bool { + lastReadTime, _, err = lock.logBuffer.LoopProcessLogData("broker", lastReadTime, 0, func() bool { lock.Mutex.Lock() lock.cond.Wait() lock.Mutex.Unlock() @@ -127,7 +128,6 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs continue } glog.Errorf("processed to %v: %v", lastReadTime, err) - time.Sleep(3127 * time.Millisecond) if err != log_buffer.ResumeError { break } @@ -141,7 +141,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (err error) { startTime = startTime.UTC() startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day()) - startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute()) + startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute()) sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() @@ -153,7 +153,8 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim dayDir := fmt.Sprintf("%s/%s", topicDir, dayEntry.Name) return filer_pb.List(broker, dayDir, "", func(hourMinuteEntry *filer_pb.Entry, isLast bool) error { if dayEntry.Name == startDate { - if strings.Compare(hourMinuteEntry.Name, startHourMinute) < 0 { + hourMinute := util.FileNameBase(hourMinuteEntry.Name) + if strings.Compare(hourMinute, startHourMinute) < 0 { return nil } } @@ -163,7 +164,7 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim // println("partition", tp.Partition, "processing", dayDir, "/", hourMinuteEntry.Name) chunkedFileReader := filer.NewChunkStreamReader(broker, hourMinuteEntry.Chunks) defer chunkedFileReader.Close() - if _, err := filer.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { + if _, err := filer.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, 0, eachLogEntryFn); err != nil { chunkedFileReader.Close() if err == io.EOF { return err diff --git a/weed/messaging/broker/broker_server.go b/weed/messaging/broker/broker_server.go index 06162471c..acf2d6d34 100644 --- a/weed/messaging/broker/broker_server.go +++ b/weed/messaging/broker/broker_server.go @@ -2,6 +2,7 @@ package broker import ( "context" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" "time" "google.golang.org/grpc" @@ -13,7 +14,7 @@ import ( ) type MessageBrokerOption struct { - Filers []string + Filers []pb.ServerAddress DefaultReplication string MaxMB int Ip string @@ -22,6 +23,7 @@ type MessageBrokerOption struct { } type MessageBroker struct { + messaging_pb.UnimplementedSeaweedMessagingServer option *MessageBrokerOption grpcDialOption grpc.DialOption topicManager *TopicManager @@ -47,7 +49,7 @@ func (broker *MessageBroker) keepConnectedToOneFiler() { for { for _, filer := range broker.option.Filers { - broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + broker.withFilerClient(false, filer, func(client filer_pb.SeaweedFilerClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() stream, err := client.KeepConnected(ctx) @@ -99,15 +101,15 @@ func (broker *MessageBroker) keepConnectedToOneFiler() { } -func (broker *MessageBroker) withFilerClient(filer string, fn func(filer_pb.SeaweedFilerClient) error) error { +func (broker *MessageBroker) withFilerClient(streamingMode bool, filer pb.ServerAddress, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithFilerClient(filer, broker.grpcDialOption, fn) + return pb.WithFilerClient(streamingMode, filer, broker.grpcDialOption, fn) } -func (broker *MessageBroker) withMasterClient(master string, fn func(client master_pb.SeaweedClient) error) error { +func (broker *MessageBroker) withMasterClient(streamingMode bool, master pb.ServerAddress, fn func(client master_pb.SeaweedClient) error) error { - return pb.WithMasterClient(master, broker.grpcDialOption, func(client master_pb.SeaweedClient) error { + return pb.WithMasterClient(streamingMode, master, broker.grpcDialOption, func(client master_pb.SeaweedClient) error { return fn(client) }) diff --git a/weed/mount/dirty_pages_chunked.go b/weed/mount/dirty_pages_chunked.go new file mode 100644 index 000000000..52308e0e5 --- /dev/null +++ b/weed/mount/dirty_pages_chunked.go @@ -0,0 +1,102 @@ +package mount + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/page_writer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" + "sync" + "time" +) + +type ChunkedDirtyPages struct { + fh *FileHandle + writeWaitGroup sync.WaitGroup + lastErr error + collection string + replication string + uploadPipeline *page_writer.UploadPipeline + hasWrites bool +} + +var ( + _ = page_writer.DirtyPages(&ChunkedDirtyPages{}) +) + +func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages { + + dirtyPages := &ChunkedDirtyPages{ + fh: fh, + } + + swapFileDir := fh.wfs.option.getTempFilePageDir() + + dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize, + dirtyPages.saveChunkedFileIntevalToStorage, fh.wfs.option.ConcurrentWriters, swapFileDir) + + return dirtyPages +} + +func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool) { + pages.hasWrites = true + + glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data))) + pages.uploadPipeline.SaveDataAt(data, offset, isSequential) + + return +} + +func (pages *ChunkedDirtyPages) FlushData() error { + if !pages.hasWrites { + return nil + } + pages.uploadPipeline.FlushAll() + if pages.lastErr != nil { + return fmt.Errorf("flush data: %v", pages.lastErr) + } + return nil +} + +func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { + if !pages.hasWrites { + return + } + return pages.uploadPipeline.MaybeReadDataAt(data, startOffset) +} + +func (pages *ChunkedDirtyPages) GetStorageOptions() (collection, replication string) { + return pages.collection, pages.replication +} + +func (pages *ChunkedDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) { + + mtime := time.Now().UnixNano() + defer cleanupFn() + + fileFullPath := pages.fh.FullPath() + fileName := fileFullPath.Name() + chunk, collection, replication, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset) + if err != nil { + glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err) + pages.lastErr = err + return + } + chunk.Mtime = mtime + pages.collection, pages.replication = collection, replication + pages.fh.AddChunks([]*filer_pb.FileChunk{chunk}) + pages.fh.entryViewCache = nil + glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size) + +} + +func (pages ChunkedDirtyPages) Destroy() { + pages.uploadPipeline.Shutdown() +} + +func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) { + pages.uploadPipeline.LockForRead(startOffset, stopOffset) +} +func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) { + pages.uploadPipeline.UnlockForRead(startOffset, stopOffset) +} diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go new file mode 100644 index 000000000..49918c104 --- /dev/null +++ b/weed/mount/filehandle.go @@ -0,0 +1,111 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "golang.org/x/exp/slices" + "io" + "sync" +) + +type FileHandleId uint64 + +type FileHandle struct { + fh FileHandleId + counter int64 + entry *filer_pb.Entry + entryLock sync.Mutex + inode uint64 + wfs *WFS + + // cache file has been written to + dirtyMetadata bool + dirtyPages *PageWriter + entryViewCache []filer.VisibleInterval + reader io.ReaderAt + contentType string + handle uint64 + sync.Mutex + + isDeleted bool +} + +func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle { + fh := &FileHandle{ + fh: handleId, + counter: 1, + inode: inode, + wfs: wfs, + } + // dirtyPages: newContinuousDirtyPages(file, writeOnly), + fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) + if entry != nil { + entry.Attributes.FileSize = filer.FileSize(entry) + } + + return fh +} + +func (fh *FileHandle) FullPath() util.FullPath { + fp, _ := fh.wfs.inodeToPath.GetPath(fh.inode) + return fp +} + +func (fh *FileHandle) GetEntry() *filer_pb.Entry { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + return fh.entry +} +func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + fh.entry = entry +} + +func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + + // find the earliest incoming chunk + newChunks := chunks + earliestChunk := newChunks[0] + for i := 1; i < len(newChunks); i++ { + if lessThan(earliestChunk, newChunks[i]) { + earliestChunk = newChunks[i] + } + } + + if fh.entry == nil { + return + } + + // pick out-of-order chunks from existing chunks + for _, chunk := range fh.entry.Chunks { + if lessThan(earliestChunk, chunk) { + chunks = append(chunks, chunk) + } + } + + // sort incoming chunks + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + return lessThan(a, b) + }) + + glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks)) + + fh.entry.Chunks = append(fh.entry.Chunks, newChunks...) + fh.entryViewCache = nil +} + +func (fh *FileHandle) Release() { + fh.dirtyPages.Destroy() +} + +func lessThan(a, b *filer_pb.FileChunk) bool { + if a.Mtime == b.Mtime { + return a.Fid.FileKey < b.Fid.FileKey + } + return a.Mtime < b.Mtime +} diff --git a/weed/mount/filehandle_map.go b/weed/mount/filehandle_map.go new file mode 100644 index 000000000..e6e2d15c9 --- /dev/null +++ b/weed/mount/filehandle_map.go @@ -0,0 +1,87 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "sync" +) + +type FileHandleToInode struct { + sync.RWMutex + nextFh FileHandleId + inode2fh map[uint64]*FileHandle + fh2inode map[FileHandleId]uint64 +} + +func NewFileHandleToInode() *FileHandleToInode { + return &FileHandleToInode{ + inode2fh: make(map[uint64]*FileHandle), + fh2inode: make(map[FileHandleId]uint64), + nextFh: 0, + } +} + +func (i *FileHandleToInode) GetFileHandle(fh FileHandleId) *FileHandle { + i.RLock() + defer i.RUnlock() + inode, found := i.fh2inode[fh] + if found { + return i.inode2fh[inode] + } + return nil +} + +func (i *FileHandleToInode) FindFileHandle(inode uint64) (fh *FileHandle, found bool) { + i.RLock() + defer i.RUnlock() + fh, found = i.inode2fh[inode] + return +} + +func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry) *FileHandle { + i.Lock() + defer i.Unlock() + fh, found := i.inode2fh[inode] + if !found { + fh = newFileHandle(wfs, i.nextFh, inode, entry) + i.nextFh++ + i.inode2fh[inode] = fh + i.fh2inode[fh.fh] = inode + } else { + fh.counter++ + } + fh.entry = entry + return fh +} + +func (i *FileHandleToInode) ReleaseByInode(inode uint64) { + i.Lock() + defer i.Unlock() + fh, found := i.inode2fh[inode] + if found { + fh.counter-- + if fh.counter <= 0 { + delete(i.inode2fh, inode) + delete(i.fh2inode, fh.fh) + fh.Release() + } + } +} +func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) { + i.Lock() + defer i.Unlock() + inode, found := i.fh2inode[fh] + if found { + fhHandle, fhFound := i.inode2fh[inode] + if !fhFound { + delete(i.fh2inode, fh) + } else { + fhHandle.counter-- + if fhHandle.counter <= 0 { + delete(i.inode2fh, inode) + delete(i.fh2inode, fhHandle.fh) + fhHandle.Release() + } + } + + } +} diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go new file mode 100644 index 000000000..88ab8612c --- /dev/null +++ b/weed/mount/filehandle_read.go @@ -0,0 +1,116 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" +) + +func (fh *FileHandle) lockForRead(startOffset int64, size int) { + fh.dirtyPages.LockForRead(startOffset, startOffset+int64(size)) +} +func (fh *FileHandle) unlockForRead(startOffset int64, size int) { + fh.dirtyPages.UnlockForRead(startOffset, startOffset+int64(size)) +} + +func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) { + maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset) + return +} + +func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { + + fileFullPath := fh.FullPath() + + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + + entry := fh.entry + if entry == nil { + return 0, io.EOF + } + + if entry.IsInRemoteOnly() { + glog.V(4).Infof("download remote entry %s", fileFullPath) + newEntry, err := fh.downloadRemoteEntry(entry) + if err != nil { + glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err) + return 0, err + } + entry = newEntry + } + + fileSize := int64(filer.FileSize(entry)) + + if fileSize == 0 { + glog.V(1).Infof("empty fh %v", fileFullPath) + return 0, io.EOF + } + + if offset+int64(len(buff)) <= int64(len(entry.Content)) { + totalRead := copy(buff, entry.Content[offset:]) + glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead) + return int64(totalRead), nil + } + + var chunkResolveErr error + if fh.entryViewCache == nil { + fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.wfs.LookupFn(), entry.Chunks, 0, fileSize) + if chunkResolveErr != nil { + return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) + } + fh.reader = nil + } + + reader := fh.reader + if reader == nil { + chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, fileSize) + glog.V(4).Infof("file handle read %s [%d,%d) from %d views", fileFullPath, offset, offset+int64(len(buff)), len(chunkViews)) + for _, chunkView := range chunkViews { + glog.V(4).Infof(" read %s [%d,%d) from chunk %+v", fileFullPath, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.FileId) + } + reader = filer.NewChunkReaderAtFromClient(fh.wfs.LookupFn(), chunkViews, fh.wfs.chunkCache, fileSize) + } + fh.reader = reader + + totalRead, err := reader.ReadAt(buff, offset) + + if err != nil && err != io.EOF { + glog.Errorf("file handle read %s: %v", fileFullPath, err) + } + + // glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err) + + return int64(totalRead), err +} + +func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) { + + fileFullPath := fh.FullPath() + dir, _ := fileFullPath.DirAndName() + + err := fh.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{ + Directory: string(dir), + Name: entry.Name, + } + + glog.V(4).Infof("download entry: %v", request) + resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request) + if err != nil { + return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err) + } + + entry = resp.Entry + + fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry)) + + return nil + }) + + return entry, err +} diff --git a/weed/mount/inode_to_path.go b/weed/mount/inode_to_path.go new file mode 100644 index 000000000..fa17a9261 --- /dev/null +++ b/weed/mount/inode_to_path.go @@ -0,0 +1,201 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fuse" + "sync" +) + +type InodeToPath struct { + sync.RWMutex + nextInodeId uint64 + inode2path map[uint64]*InodeEntry + path2inode map[util.FullPath]uint64 +} +type InodeEntry struct { + util.FullPath + nlookup uint64 + isDirectory bool + isChildrenCached bool +} + +func NewInodeToPath(root util.FullPath) *InodeToPath { + t := &InodeToPath{ + inode2path: make(map[uint64]*InodeEntry), + path2inode: make(map[util.FullPath]uint64), + } + t.inode2path[1] = &InodeEntry{root, 1, true, false} + t.path2inode[root] = 1 + return t +} + +func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory bool, isHardlink bool, possibleInode uint64, isLookup bool) uint64 { + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if !found { + if possibleInode == 0 { + inode = path.AsInode(unixTime) + } else { + inode = possibleInode + } + if !isHardlink { + for _, found := i.inode2path[inode]; found; inode++ { + _, found = i.inode2path[inode] + } + } + } + i.path2inode[path] = inode + + if _, found := i.inode2path[inode]; found { + if isLookup { + i.inode2path[inode].nlookup++ + } + } else { + if !isLookup { + i.inode2path[inode] = &InodeEntry{path, 0, isDirectory, false} + } else { + i.inode2path[inode] = &InodeEntry{path, 1, isDirectory, false} + } + } + + return inode +} + +func (i *InodeToPath) AllocateInode(path util.FullPath, unixTime int64) uint64 { + if path == "/" { + return 1 + } + i.Lock() + defer i.Unlock() + inode := path.AsInode(unixTime) + for _, found := i.inode2path[inode]; found; inode++ { + _, found = i.inode2path[inode] + } + return inode +} + +func (i *InodeToPath) GetInode(path util.FullPath) uint64 { + if path == "/" { + return 1 + } + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if !found { + // glog.Fatalf("GetInode unknown inode for %s", path) + // this could be the parent for mount point + } + return inode +} + +func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) { + i.RLock() + defer i.RUnlock() + path, found := i.inode2path[inode] + if !found { + return "", fuse.ENOENT + } + return path.FullPath, fuse.OK +} + +func (i *InodeToPath) HasPath(path util.FullPath) bool { + i.RLock() + defer i.RUnlock() + _, found := i.path2inode[path] + return found +} + +func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) { + i.RLock() + defer i.RUnlock() + inode, found := i.path2inode[fullpath] + if !found { + glog.Fatalf("MarkChildrenCached not found inode %v", fullpath) + } + path, found := i.inode2path[inode] + path.isChildrenCached = true +} + +func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool { + i.RLock() + defer i.RUnlock() + inode, found := i.path2inode[fullpath] + if !found { + return false + } + path, found := i.inode2path[inode] + if found { + return path.isChildrenCached + } + return false +} + +func (i *InodeToPath) HasInode(inode uint64) bool { + if inode == 1 { + return true + } + i.RLock() + defer i.RUnlock() + _, found := i.inode2path[inode] + return found +} + +func (i *InodeToPath) RemovePath(path util.FullPath) { + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if found { + delete(i.path2inode, path) + delete(i.inode2path, inode) + } +} + +func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (replacedInode uint64) { + i.Lock() + defer i.Unlock() + sourceInode, sourceFound := i.path2inode[sourcePath] + targetInode, targetFound := i.path2inode[targetPath] + if targetFound { + delete(i.inode2path, targetInode) + delete(i.path2inode, targetPath) + } + if sourceFound { + delete(i.path2inode, sourcePath) + i.path2inode[targetPath] = sourceInode + } else { + // it is possible some source folder items has not been visited before + // so no need to worry about their source inodes + return + } + if entry, entryFound := i.inode2path[sourceInode]; entryFound { + entry.FullPath = targetPath + entry.isChildrenCached = false + if !targetFound { + entry.nlookup++ + } + } else { + glog.Errorf("MovePath %s to %s: sourceInode %d not found", sourcePath, targetPath, sourceInode) + } + return targetInode +} + +func (i *InodeToPath) Forget(inode, nlookup uint64, onForgetDir func(dir util.FullPath)) { + i.Lock() + path, found := i.inode2path[inode] + if found { + path.nlookup -= nlookup + if path.nlookup <= 0 { + delete(i.path2inode, path.FullPath) + delete(i.inode2path, inode) + } + } + i.Unlock() + if found { + if path.isDirectory && path.nlookup <= 0 && onForgetDir != nil { + path.isChildrenCached = false + onForgetDir(path.FullPath) + } + } +} diff --git a/weed/filesys/meta_cache/cache_config.go b/weed/mount/meta_cache/cache_config.go similarity index 100% rename from weed/filesys/meta_cache/cache_config.go rename to weed/mount/meta_cache/cache_config.go diff --git a/weed/filesys/meta_cache/id_mapper.go b/weed/mount/meta_cache/id_mapper.go similarity index 100% rename from weed/filesys/meta_cache/id_mapper.go rename to weed/mount/meta_cache/id_mapper.go diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go similarity index 62% rename from weed/filesys/meta_cache/meta_cache.go rename to weed/mount/meta_cache/meta_cache.go index 69d1655ee..8c434787a 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/mount/meta_cache/meta_cache.go @@ -2,12 +2,11 @@ package meta_cache import ( "context" - "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/leveldb" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/util/bounded_tree" "os" ) @@ -15,20 +14,25 @@ import ( // e.g. fill fileId field for chunks type MetaCache struct { + root util.FullPath localStore filer.VirtualFilerStore // sync.RWMutex - visitedBoundary *bounded_tree.BoundedTree - uidGidMapper *UidGidMapper - invalidateFunc func(util.FullPath) + uidGidMapper *UidGidMapper + markCachedFn func(fullpath util.FullPath) + isCachedFn func(fullpath util.FullPath) bool + invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry) } -func NewMetaCache(dbFolder string, baseDir util.FullPath, uidGidMapper *UidGidMapper, invalidateFunc func(util.FullPath)) *MetaCache { +func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, + markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry)) *MetaCache { return &MetaCache{ - localStore: openMetaStore(dbFolder), - visitedBoundary: bounded_tree.NewBoundedTree(baseDir), - uidGidMapper: uidGidMapper, - invalidateFunc: func(fullpath util.FullPath) { - invalidateFunc(fullpath) + root: root, + localStore: openMetaStore(dbFolder), + markCachedFn: markCachedFn, + isCachedFn: isCachedFn, + uidGidMapper: uidGidMapper, + invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) { + invalidateFunc(fullpath, entry) }, } } @@ -61,20 +65,26 @@ func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) erro return mc.localStore.InsertEntry(ctx, entry) } -func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error { +func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry, shouldDeleteChunks bool) error { //mc.Lock() //defer mc.Unlock() oldDir, _ := oldPath.DirAndName() - if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) { + if mc.isCachedFn(util.FullPath(oldDir)) { if oldPath != "" { if newEntry != nil && oldPath == newEntry.FullPath { // skip the unnecessary deletion // leave the update to the following InsertEntry operation } else { glog.V(3).Infof("DeleteEntry %s", oldPath) - if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil { - return err + if shouldDeleteChunks { + if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil { + return err + } + } else { + if err := mc.localStore.DeleteOneEntrySkipHardlink(ctx, oldPath); err != nil { + return err + } } } } @@ -84,7 +94,7 @@ func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath uti if newEntry != nil { newDir, _ := newEntry.DirAndName() - if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) { + if mc.isCachedFn(util.FullPath(newDir)) { glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name()) if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil { return err @@ -111,18 +121,30 @@ func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *fi return } +func (mc *MetaCache) DeleteEntrySkipHardlink(ctx context.Context, fp util.FullPath) (err error) { + //mc.Lock() + //defer mc.Unlock() + return mc.localStore.DeleteOneEntrySkipHardlink(ctx, fp) +} + func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) { //mc.Lock() //defer mc.Unlock() return mc.localStore.DeleteEntry(ctx, fp) } +func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { + //mc.Lock() + //defer mc.Unlock() + return mc.localStore.DeleteFolderChildren(ctx, fp) +} func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error { //mc.RLock() //defer mc.RUnlock() - if !mc.visitedBoundary.HasVisited(dirPath) { - return fmt.Errorf("unsynchronized dir: %v", dirPath) + if !mc.isCachedFn(dirPath) { + // if this request comes after renaming, it should be fine + glog.Warningf("unsynchronized dir: %v", dirPath) } _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool { @@ -144,3 +166,10 @@ func (mc *MetaCache) Shutdown() { func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) { entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid) } + +func (mc *MetaCache) Debug() { + if debuggable, ok := mc.localStore.(filer.Debuggable); ok { + println("start debugging") + debuggable.Debug(os.Stderr) + } +} diff --git a/weed/mount/meta_cache/meta_cache_init.go b/weed/mount/meta_cache/meta_cache_init.go new file mode 100644 index 000000000..f360f1f2b --- /dev/null +++ b/weed/mount/meta_cache/meta_cache_init.go @@ -0,0 +1,70 @@ +package meta_cache + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath) error { + + currentPath := dirPath + + for { + + // the directory children are already cached + // so no need for this and upper directories + if mc.isCachedFn(currentPath) { + return nil + } + + if err := doEnsureVisited(mc, client, currentPath); err != nil { + return err + } + + // continue to parent directory + if currentPath != mc.root { + parent, _ := currentPath.DirAndName() + currentPath = util.FullPath(parent) + } else { + break + } + } + + return nil + +} + +func doEnsureVisited(mc *MetaCache, client filer_pb.FilerClient, path util.FullPath) error { + + glog.V(4).Infof("ReadDirAllEntries %s ...", path) + + err := util.Retry("ReadDirAllEntries", func() error { + return filer_pb.ReadDirAllEntries(client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error { + entry := filer.FromPbEntry(string(path), pbEntry) + if IsHiddenSystemEntry(string(path), entry.Name()) { + return nil + } + if err := mc.doInsertEntry(context.Background(), entry); err != nil { + glog.V(0).Infof("read %s: %v", entry.FullPath, err) + return err + } + return nil + }) + }) + + if err != nil { + err = fmt.Errorf("list %s: %v", path, err) + } else { + mc.markCachedFn(path) + } + return err +} + +func IsHiddenSystemEntry(dir, name string) bool { + return dir == "/" && (name == "topics" || name == "etc") +} diff --git a/weed/filesys/meta_cache/meta_cache_subscribe.go b/weed/mount/meta_cache/meta_cache_subscribe.go similarity index 81% rename from weed/filesys/meta_cache/meta_cache_subscribe.go rename to weed/mount/meta_cache/meta_cache_subscribe.go index 2099cf1f8..c8ccdd375 100644 --- a/weed/filesys/meta_cache/meta_cache_subscribe.go +++ b/weed/mount/meta_cache/meta_cache_subscribe.go @@ -36,20 +36,20 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil glog.V(4).Infof("creating %v", key) newEntry = filer.FromPbEntry(dir, message.NewEntry) } - err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry) + err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry, message.DeleteChunks) if err == nil { if message.OldEntry != nil && message.NewEntry != nil { oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) - mc.invalidateFunc(oldKey) + mc.invalidateFunc(oldKey, message.OldEntry) if message.OldEntry.Name != message.NewEntry.Name { newKey := util.NewFullPath(dir, message.NewEntry.Name) - mc.invalidateFunc(newKey) + mc.invalidateFunc(newKey, message.NewEntry) } - } else if message.OldEntry == nil && message.NewEntry != nil { + } else if filer_pb.IsCreate(resp) { // no need to invaalidate - } else if message.OldEntry != nil && message.NewEntry == nil { + } else if filer_pb.IsDelete(resp) { oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) - mc.invalidateFunc(oldKey) + mc.invalidateFunc(oldKey, message.OldEntry) } } @@ -58,7 +58,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } util.RetryForever("followMetaUpdates", func() error { - return pb.WithFilerClientFollowMetadata(client, "mount", dir, lastTsNs, selfSignature, processEventFn, true) + return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, dir, &lastTsNs, 0, selfSignature, processEventFn, pb.FatalOnError) }, func(err error) bool { glog.Errorf("follow metadata updates: %v", err) return true diff --git a/weed/mount/page_writer.go b/weed/mount/page_writer.go new file mode 100644 index 000000000..016c4841a --- /dev/null +++ b/weed/mount/page_writer.go @@ -0,0 +1,96 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/page_writer" +) + +type PageWriter struct { + fh *FileHandle + collection string + replication string + chunkSize int64 + writerPattern *WriterPattern + + randomWriter page_writer.DirtyPages +} + +var ( + _ = page_writer.DirtyPages(&PageWriter{}) +) + +func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter { + pw := &PageWriter{ + fh: fh, + chunkSize: chunkSize, + writerPattern: NewWriterPattern(chunkSize), + randomWriter: newMemoryChunkPages(fh, chunkSize), + } + return pw +} + +func (pw *PageWriter) AddPage(offset int64, data []byte, isSequentail bool) { + + glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data))) + + chunkIndex := offset / pw.chunkSize + for i := chunkIndex; len(data) > 0; i++ { + writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset) + pw.addToOneChunk(i, offset, data[:writeSize], isSequentail) + offset += writeSize + data = data[writeSize:] + } +} + +func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool) { + pw.randomWriter.AddPage(offset, data, isSequential) +} + +func (pw *PageWriter) FlushData() error { + return pw.randomWriter.FlushData() +} + +func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64) (maxStop int64) { + glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.fh, offset, offset+int64(len(data))) + + chunkIndex := offset / pw.chunkSize + for i := chunkIndex; len(data) > 0; i++ { + readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset) + + maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset) + + offset += readSize + data = data[readSize:] + } + + return +} + +func (pw *PageWriter) GetStorageOptions() (collection, replication string) { + return pw.randomWriter.GetStorageOptions() +} + +func (pw *PageWriter) LockForRead(startOffset, stopOffset int64) { + pw.randomWriter.LockForRead(startOffset, stopOffset) +} + +func (pw *PageWriter) UnlockForRead(startOffset, stopOffset int64) { + pw.randomWriter.UnlockForRead(startOffset, stopOffset) +} + +func (pw *PageWriter) Destroy() { + pw.randomWriter.Destroy() +} + +func max(x, y int64) int64 { + if x > y { + return x + } + return y +} +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} diff --git a/weed/mount/page_writer/chunk_interval_list.go b/weed/mount/page_writer/chunk_interval_list.go new file mode 100644 index 000000000..e6dc5d1f5 --- /dev/null +++ b/weed/mount/page_writer/chunk_interval_list.go @@ -0,0 +1,115 @@ +package page_writer + +import "math" + +// ChunkWrittenInterval mark one written interval within one page chunk +type ChunkWrittenInterval struct { + StartOffset int64 + stopOffset int64 + prev *ChunkWrittenInterval + next *ChunkWrittenInterval +} + +func (interval *ChunkWrittenInterval) Size() int64 { + return interval.stopOffset - interval.StartOffset +} + +func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool { + return interval.stopOffset-interval.StartOffset == chunkSize +} + +// ChunkWrittenIntervalList mark written intervals within one page chunk +type ChunkWrittenIntervalList struct { + head *ChunkWrittenInterval + tail *ChunkWrittenInterval +} + +func newChunkWrittenIntervalList() *ChunkWrittenIntervalList { + list := &ChunkWrittenIntervalList{ + head: &ChunkWrittenInterval{ + StartOffset: -1, + stopOffset: -1, + }, + tail: &ChunkWrittenInterval{ + StartOffset: math.MaxInt64, + stopOffset: math.MaxInt64, + }, + } + list.head.next = list.tail + list.tail.prev = list.head + return list +} + +func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset int64) { + interval := &ChunkWrittenInterval{ + StartOffset: startOffset, + stopOffset: stopOffset, + } + list.addInterval(interval) +} + +func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool { + return list.size() == 1 && list.head.next.isComplete(chunkSize) +} +func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) { + for t := list.head; t != nil; t = t.next { + writtenByteCount += t.Size() + } + return +} + +func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) { + + p := list.head + for ; p.next != nil && p.next.StartOffset <= interval.StartOffset; p = p.next { + } + q := list.tail + for ; q.prev != nil && q.prev.stopOffset >= interval.stopOffset; q = q.prev { + } + + if interval.StartOffset <= p.stopOffset && q.StartOffset <= interval.stopOffset { + // merge p and q together + p.stopOffset = q.stopOffset + unlinkNodesBetween(p, q.next) + return + } + if interval.StartOffset <= p.stopOffset { + // merge new interval into p + p.stopOffset = interval.stopOffset + unlinkNodesBetween(p, q) + return + } + if q.StartOffset <= interval.stopOffset { + // merge new interval into q + q.StartOffset = interval.StartOffset + unlinkNodesBetween(p, q) + return + } + + // add the new interval between p and q + unlinkNodesBetween(p, q) + p.next = interval + interval.prev = p + q.prev = interval + interval.next = q + +} + +// unlinkNodesBetween remove all nodes after start and before stop, exclusive +func unlinkNodesBetween(start *ChunkWrittenInterval, stop *ChunkWrittenInterval) { + if start.next == stop { + return + } + start.next.prev = nil + start.next = stop + stop.prev.next = nil + stop.prev = start +} + +func (list *ChunkWrittenIntervalList) size() int { + var count int + for t := list.head; t != nil; t = t.next { + count++ + } + return count - 2 +} diff --git a/weed/mount/page_writer/chunk_interval_list_test.go b/weed/mount/page_writer/chunk_interval_list_test.go new file mode 100644 index 000000000..b22f5eb5d --- /dev/null +++ b/weed/mount/page_writer/chunk_interval_list_test.go @@ -0,0 +1,49 @@ +package page_writer + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_PageChunkWrittenIntervalList(t *testing.T) { + list := newChunkWrittenIntervalList() + + assert.Equal(t, 0, list.size(), "empty list") + + list.MarkWritten(0, 5) + assert.Equal(t, 1, list.size(), "one interval") + + list.MarkWritten(0, 5) + assert.Equal(t, 1, list.size(), "duplicated interval2") + + list.MarkWritten(95, 100) + assert.Equal(t, 2, list.size(), "two intervals") + + list.MarkWritten(50, 60) + assert.Equal(t, 3, list.size(), "three intervals") + + list.MarkWritten(50, 55) + assert.Equal(t, 3, list.size(), "three intervals merge") + + list.MarkWritten(40, 50) + assert.Equal(t, 3, list.size(), "three intervals grow forward") + + list.MarkWritten(50, 65) + assert.Equal(t, 3, list.size(), "three intervals grow backward") + + list.MarkWritten(70, 80) + assert.Equal(t, 4, list.size(), "four intervals") + + list.MarkWritten(60, 70) + assert.Equal(t, 3, list.size(), "three intervals merged") + + list.MarkWritten(59, 71) + assert.Equal(t, 3, list.size(), "covered three intervals") + + list.MarkWritten(5, 59) + assert.Equal(t, 2, list.size(), "covered two intervals") + + list.MarkWritten(70, 99) + assert.Equal(t, 1, list.size(), "covered one intervals") + +} diff --git a/weed/mount/page_writer/dirty_pages.go b/weed/mount/page_writer/dirty_pages.go new file mode 100644 index 000000000..c16cee47a --- /dev/null +++ b/weed/mount/page_writer/dirty_pages.go @@ -0,0 +1,30 @@ +package page_writer + +type DirtyPages interface { + AddPage(offset int64, data []byte, isSequential bool) + FlushData() error + ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) + GetStorageOptions() (collection, replication string) + Destroy() + LockForRead(startOffset, stopOffset int64) + UnlockForRead(startOffset, stopOffset int64) +} + +func max(x, y int64) int64 { + if x > y { + return x + } + return y +} +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} +func minInt(x, y int) int { + if x < y { + return x + } + return y +} diff --git a/weed/mount/page_writer/page_chunk.go b/weed/mount/page_writer/page_chunk.go new file mode 100644 index 000000000..4e8f31425 --- /dev/null +++ b/weed/mount/page_writer/page_chunk.go @@ -0,0 +1,16 @@ +package page_writer + +import ( + "io" +) + +type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupFn func()) + +type PageChunk interface { + FreeResource() + WriteDataAt(src []byte, offset int64) (n int) + ReadDataAt(p []byte, off int64) (maxStop int64) + IsComplete() bool + WrittenSize() int64 + SaveContent(saveFn SaveToStorageFunc) +} diff --git a/weed/mount/page_writer/page_chunk_mem.go b/weed/mount/page_writer/page_chunk_mem.go new file mode 100644 index 000000000..52db6d4f9 --- /dev/null +++ b/weed/mount/page_writer/page_chunk_mem.go @@ -0,0 +1,74 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/mem" + "sync/atomic" +) + +var ( + _ = PageChunk(&MemChunk{}) + + memChunkCounter int64 +) + +type MemChunk struct { + buf []byte + usage *ChunkWrittenIntervalList + chunkSize int64 + logicChunkIndex LogicChunkIndex +} + +func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk { + atomic.AddInt64(&memChunkCounter, 1) + return &MemChunk{ + logicChunkIndex: logicChunkIndex, + chunkSize: chunkSize, + buf: mem.Allocate(int(chunkSize)), + usage: newChunkWrittenIntervalList(), + } +} + +func (mc *MemChunk) FreeResource() { + atomic.AddInt64(&memChunkCounter, -1) + mem.Free(mc.buf) +} + +func (mc *MemChunk) WriteDataAt(src []byte, offset int64) (n int) { + innerOffset := offset % mc.chunkSize + n = copy(mc.buf[innerOffset:], src) + mc.usage.MarkWritten(innerOffset, innerOffset+int64(n)) + return +} + +func (mc *MemChunk) ReadDataAt(p []byte, off int64) (maxStop int64) { + memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize + for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { + logicStart := max(off, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset) + logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset) + if logicStart < logicStop { + copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset]) + maxStop = max(maxStop, logicStop) + } + } + return +} + +func (mc *MemChunk) IsComplete() bool { + return mc.usage.IsComplete(mc.chunkSize) +} + +func (mc *MemChunk) WrittenSize() int64 { + return mc.usage.WrittenSize() +} + +func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) { + if saveFn == nil { + return + } + for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { + reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset]) + saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), func() { + }) + } +} diff --git a/weed/mount/page_writer/page_chunk_swapfile.go b/weed/mount/page_writer/page_chunk_swapfile.go new file mode 100644 index 000000000..b56589bfc --- /dev/null +++ b/weed/mount/page_writer/page_chunk_swapfile.go @@ -0,0 +1,136 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/mem" + "os" + "sync" +) + +var ( + _ = PageChunk(&SwapFileChunk{}) +) + +type ActualChunkIndex int + +type SwapFile struct { + dir string + file *os.File + logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex + logicToActualChunkIndexLock sync.Mutex + chunkSize int64 + freeActualChunkList []ActualChunkIndex +} + +type SwapFileChunk struct { + swapfile *SwapFile + usage *ChunkWrittenIntervalList + logicChunkIndex LogicChunkIndex + actualChunkIndex ActualChunkIndex +} + +func NewSwapFile(dir string, chunkSize int64) *SwapFile { + return &SwapFile{ + dir: dir, + file: nil, + logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex), + chunkSize: chunkSize, + } +} +func (sf *SwapFile) FreeResource() { + if sf.file != nil { + sf.file.Close() + os.Remove(sf.file.Name()) + } +} + +func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) { + if sf.file == nil { + var err error + sf.file, err = os.CreateTemp(sf.dir, "") + if err != nil { + glog.Errorf("create swap file: %v", err) + return nil + } + } + sf.logicToActualChunkIndexLock.Lock() + defer sf.logicToActualChunkIndexLock.Unlock() + actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex] + if !found { + if len(sf.freeActualChunkList) > 0 { + actualChunkIndex = sf.freeActualChunkList[0] + sf.freeActualChunkList = sf.freeActualChunkList[1:] + } else { + actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex)) + } + sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex + } + + return &SwapFileChunk{ + swapfile: sf, + usage: newChunkWrittenIntervalList(), + logicChunkIndex: logicChunkIndex, + actualChunkIndex: actualChunkIndex, + } +} + +func (sc *SwapFileChunk) FreeResource() { + sc.swapfile.logicToActualChunkIndexLock.Lock() + defer sc.swapfile.logicToActualChunkIndexLock.Unlock() + + sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex) + delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex) +} + +func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) { + innerOffset := offset % sc.swapfile.chunkSize + var err error + n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset) + if err == nil { + sc.usage.MarkWritten(innerOffset, innerOffset+int64(n)) + } else { + glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err) + } + return +} + +func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) { + chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize + for t := sc.usage.head.next; t != sc.usage.tail; t = t.next { + logicStart := max(off, chunkStartOffset+t.StartOffset) + logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset) + if logicStart < logicStop { + actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize + if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil { + glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err) + break + } + maxStop = max(maxStop, logicStop) + } + } + return +} + +func (sc *SwapFileChunk) IsComplete() bool { + return sc.usage.IsComplete(sc.swapfile.chunkSize) +} + +func (sc *SwapFileChunk) WrittenSize() int64 { + return sc.usage.WrittenSize() +} + +func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) { + if saveFn == nil { + return + } + for t := sc.usage.head.next; t != sc.usage.tail; t = t.next { + data := mem.Allocate(int(t.Size())) + sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize) + reader := util.NewBytesReader(data) + saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() { + }) + mem.Free(data) + } + sc.usage = newChunkWrittenIntervalList() +} diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go new file mode 100644 index 000000000..0c7446cad --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline.go @@ -0,0 +1,196 @@ +package page_writer + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "sync" + "sync/atomic" + "time" +) + +type LogicChunkIndex int + +type UploadPipeline struct { + filepath util.FullPath + ChunkSize int64 + writableChunks map[LogicChunkIndex]PageChunk + writableChunksLock sync.Mutex + sealedChunks map[LogicChunkIndex]*SealedChunk + sealedChunksLock sync.Mutex + uploaders *util.LimitedConcurrentExecutor + uploaderCount int32 + uploaderCountCond *sync.Cond + saveToStorageFn SaveToStorageFunc + activeReadChunks map[LogicChunkIndex]int + activeReadChunksLock sync.Mutex + writableChunkLimit int + swapFile *SwapFile +} + +type SealedChunk struct { + chunk PageChunk + referenceCounter int // track uploading or reading processes +} + +func (sc *SealedChunk) FreeReference(messageOnFree string) { + sc.referenceCounter-- + if sc.referenceCounter == 0 { + glog.V(4).Infof("Free sealed chunk: %s", messageOnFree) + sc.chunk.FreeResource() + } +} + +func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string) *UploadPipeline { + return &UploadPipeline{ + ChunkSize: chunkSize, + writableChunks: make(map[LogicChunkIndex]PageChunk), + sealedChunks: make(map[LogicChunkIndex]*SealedChunk), + uploaders: writers, + uploaderCountCond: sync.NewCond(&sync.Mutex{}), + saveToStorageFn: saveToStorageFn, + activeReadChunks: make(map[LogicChunkIndex]int), + writableChunkLimit: bufferChunkLimit, + swapFile: NewSwapFile(swapFileDir, chunkSize), + } +} + +func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n int) { + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + + logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) + + pageChunk, found := up.writableChunks[logicChunkIndex] + if !found { + if len(up.writableChunks) > up.writableChunkLimit { + // if current file chunks is over the per file buffer count limit + fullestChunkIndex, fullness := LogicChunkIndex(-1), int64(0) + for lci, mc := range up.writableChunks { + chunkFullness := mc.WrittenSize() + if fullness < chunkFullness { + fullestChunkIndex = lci + fullness = chunkFullness + } + } + up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex) + delete(up.writableChunks, fullestChunkIndex) + // fmt.Printf("flush chunk %d with %d bytes written\n", logicChunkIndex, fullness) + } + if isSequential && + len(up.writableChunks) < up.writableChunkLimit && + atomic.LoadInt64(&memChunkCounter) < 4*int64(up.writableChunkLimit) { + pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize) + } else { + pageChunk = up.swapFile.NewTempFileChunk(logicChunkIndex) + } + up.writableChunks[logicChunkIndex] = pageChunk + } + n = pageChunk.WriteDataAt(p, off) + up.maybeMoveToSealed(pageChunk, logicChunkIndex) + + return +} + +func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) { + logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) + + // read from sealed chunks first + up.sealedChunksLock.Lock() + sealedChunk, found := up.sealedChunks[logicChunkIndex] + if found { + sealedChunk.referenceCounter++ + } + up.sealedChunksLock.Unlock() + if found { + maxStop = sealedChunk.chunk.ReadDataAt(p, off) + glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop) + sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", up.filepath, logicChunkIndex)) + } + + // read from writable chunks last + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + writableChunk, found := up.writableChunks[logicChunkIndex] + if !found { + return + } + writableMaxStop := writableChunk.ReadDataAt(p, off) + glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop) + maxStop = max(maxStop, writableMaxStop) + + return +} + +func (up *UploadPipeline) FlushAll() { + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + + for logicChunkIndex, memChunk := range up.writableChunks { + up.moveToSealed(memChunk, logicChunkIndex) + } + + up.waitForCurrentWritersToComplete() +} + +func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) { + if memChunk.IsComplete() { + up.moveToSealed(memChunk, logicChunkIndex) + } +} + +func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) { + atomic.AddInt32(&up.uploaderCount, 1) + glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount) + + up.sealedChunksLock.Lock() + + if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found { + oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex)) + } + sealedChunk := &SealedChunk{ + chunk: memChunk, + referenceCounter: 1, // default 1 is for uploading process + } + up.sealedChunks[logicChunkIndex] = sealedChunk + delete(up.writableChunks, logicChunkIndex) + + up.sealedChunksLock.Unlock() + + up.uploaders.Execute(func() { + // first add to the file chunks + sealedChunk.chunk.SaveContent(up.saveToStorageFn) + + // notify waiting process + atomic.AddInt32(&up.uploaderCount, -1) + glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, up.uploaderCount+1, up.uploaderCount) + // Lock and Unlock are not required, + // but it may signal multiple times during one wakeup, + // and the waiting goroutine may miss some of them! + up.uploaderCountCond.L.Lock() + up.uploaderCountCond.Broadcast() + up.uploaderCountCond.L.Unlock() + + // wait for readers + for up.IsLocked(logicChunkIndex) { + time.Sleep(59 * time.Millisecond) + } + + // then remove from sealed chunks + up.sealedChunksLock.Lock() + defer up.sealedChunksLock.Unlock() + delete(up.sealedChunks, logicChunkIndex) + sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex)) + + }) +} + +func (up *UploadPipeline) Shutdown() { + up.swapFile.FreeResource() + + up.sealedChunksLock.Lock() + defer up.sealedChunksLock.Unlock() + for logicChunkIndex, sealedChunk := range up.sealedChunks { + sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex)) + } +} diff --git a/weed/mount/page_writer/upload_pipeline_lock.go b/weed/mount/page_writer/upload_pipeline_lock.go new file mode 100644 index 000000000..47a40ba37 --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline_lock.go @@ -0,0 +1,63 @@ +package page_writer + +import ( + "sync/atomic" +) + +func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) { + startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize) + stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize) + if stopOffset%up.ChunkSize > 0 { + stopLogicChunkIndex += 1 + } + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { + if count, found := up.activeReadChunks[i]; found { + up.activeReadChunks[i] = count + 1 + } else { + up.activeReadChunks[i] = 1 + } + } +} + +func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) { + startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize) + stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize) + if stopOffset%up.ChunkSize > 0 { + stopLogicChunkIndex += 1 + } + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { + if count, found := up.activeReadChunks[i]; found { + if count == 1 { + delete(up.activeReadChunks, i) + } else { + up.activeReadChunks[i] = count - 1 + } + } + } +} + +func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool { + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + if count, found := up.activeReadChunks[logicChunkIndex]; found { + return count > 0 + } + return false +} + +func (up *UploadPipeline) waitForCurrentWritersToComplete() { + up.uploaderCountCond.L.Lock() + t := int32(100) + for { + t = atomic.LoadInt32(&up.uploaderCount) + if t <= 0 { + break + } + up.uploaderCountCond.Wait() + } + up.uploaderCountCond.L.Unlock() +} diff --git a/weed/mount/page_writer/upload_pipeline_test.go b/weed/mount/page_writer/upload_pipeline_test.go new file mode 100644 index 000000000..f130c97c1 --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline_test.go @@ -0,0 +1,47 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/util" + "testing" +) + +func TestUploadPipeline(t *testing.T) { + + uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16, "") + + writeRange(uploadPipeline, 0, 131072) + writeRange(uploadPipeline, 131072, 262144) + writeRange(uploadPipeline, 262144, 1025536) + + confirmRange(t, uploadPipeline, 0, 1025536) + + writeRange(uploadPipeline, 1025536, 1296896) + + confirmRange(t, uploadPipeline, 1025536, 1296896) + + writeRange(uploadPipeline, 1296896, 2162688) + + confirmRange(t, uploadPipeline, 1296896, 2162688) + + confirmRange(t, uploadPipeline, 1296896, 2162688) +} + +// startOff and stopOff must be divided by 4 +func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) { + p := make([]byte, 4) + for i := startOff / 4; i < stopOff/4; i += 4 { + util.Uint32toBytes(p, uint32(i)) + uploadPipeline.SaveDataAt(p, i, false) + } +} + +func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) { + p := make([]byte, 4) + for i := startOff; i < stopOff/4; i += 4 { + uploadPipeline.MaybeReadDataAt(p, i) + x := util.BytesToUint32(p) + if x != uint32(i) { + t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4) + } + } +} diff --git a/weed/mount/page_writer_pattern.go b/weed/mount/page_writer_pattern.go new file mode 100644 index 000000000..665056b36 --- /dev/null +++ b/weed/mount/page_writer_pattern.go @@ -0,0 +1,44 @@ +package mount + +type WriterPattern struct { + isStreaming bool + lastWriteOffset int64 + chunkSize int64 +} + +// For streaming write: only cache the first chunk +// For random write: fall back to temp file approach +// writes can only change from streaming mode to non-streaming mode + +func NewWriterPattern(chunkSize int64) *WriterPattern { + return &WriterPattern{ + isStreaming: true, + lastWriteOffset: -1, + chunkSize: chunkSize, + } +} + +func (rp *WriterPattern) MonitorWriteAt(offset int64, size int) { + if rp.lastWriteOffset > offset { + rp.isStreaming = false + } + if rp.lastWriteOffset == -1 { + if offset != 0 { + rp.isStreaming = false + } + } + rp.lastWriteOffset = offset +} + +func (rp *WriterPattern) IsStreamingMode() bool { + return rp.isStreaming +} + +func (rp *WriterPattern) IsRandomMode() bool { + return !rp.isStreaming +} + +func (rp *WriterPattern) Reset() { + rp.isStreaming = true + rp.lastWriteOffset = -1 +} diff --git a/weed/mount/unmount/unmount.go b/weed/mount/unmount/unmount.go new file mode 100644 index 000000000..c481d8030 --- /dev/null +++ b/weed/mount/unmount/unmount.go @@ -0,0 +1,6 @@ +package unmount + +// Unmount tries to unmount the filesystem mounted at dir. +func Unmount(dir string) error { + return unmount(dir) +} diff --git a/weed/mount/unmount/unmount_linux.go b/weed/mount/unmount/unmount_linux.go new file mode 100644 index 000000000..e55d48f86 --- /dev/null +++ b/weed/mount/unmount/unmount_linux.go @@ -0,0 +1,21 @@ +package unmount + +import ( + "bytes" + "errors" + "os/exec" +) + +func unmount(dir string) error { + cmd := exec.Command("fusermount", "-u", dir) + output, err := cmd.CombinedOutput() + if err != nil { + if len(output) > 0 { + output = bytes.TrimRight(output, "\n") + msg := err.Error() + ": " + string(output) + err = errors.New(msg) + } + return err + } + return nil +} diff --git a/weed/mount/unmount/unmount_std.go b/weed/mount/unmount/unmount_std.go new file mode 100644 index 000000000..410eb1235 --- /dev/null +++ b/weed/mount/unmount/unmount_std.go @@ -0,0 +1,18 @@ +//go:build !linux && !windows +// +build !linux,!windows + +package unmount + +import ( + "os" + "syscall" +) + +func unmount(dir string) error { + err := syscall.Unmount(dir, 0) + if err != nil { + err = &os.PathError{Op: "unmount", Path: dir, Err: err} + return err + } + return nil +} diff --git a/weed/mount/unmount/unmount_unsupported.go b/weed/mount/unmount/unmount_unsupported.go new file mode 100644 index 000000000..d0a94cc4a --- /dev/null +++ b/weed/mount/unmount/unmount_unsupported.go @@ -0,0 +1,8 @@ +//go:build windows +// +build windows + +package unmount + +func unmount(dir string) error { + return nil +} diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go new file mode 100644 index 000000000..584174202 --- /dev/null +++ b/weed/mount/weedfs.go @@ -0,0 +1,200 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/mount_pb" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" + "github.com/chrislusf/seaweedfs/weed/util/grace" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "github.com/hanwen/go-fuse/v2/fuse" + "google.golang.org/grpc" + "math/rand" + "os" + "path" + "path/filepath" + "time" + + "github.com/hanwen/go-fuse/v2/fs" +) + +type Option struct { + MountDirectory string + FilerAddresses []pb.ServerAddress + filerIndex int + GrpcDialOption grpc.DialOption + FilerMountRootPath string + Collection string + Replication string + TtlSec int32 + DiskType types.DiskType + ChunkSizeLimit int64 + ConcurrentWriters int + CacheDir string + CacheSizeMB int64 + DataCenter string + Umask os.FileMode + Quota int64 + DisableXAttr bool + + MountUid uint32 + MountGid uint32 + MountMode os.FileMode + MountCtime time.Time + MountMtime time.Time + MountParentInode uint64 + + VolumeServerAccess string // how to access volume servers + Cipher bool // whether encrypt data on volume server + UidGidMapper *meta_cache.UidGidMapper + + uniqueCacheDir string + uniqueCacheTempPageDir string +} + +type WFS struct { + // https://dl.acm.org/doi/fullHtml/10.1145/3310148 + // follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go + fuse.RawFileSystem + mount_pb.UnimplementedSeaweedMountServer + fs.Inode + option *Option + metaCache *meta_cache.MetaCache + stats statsCache + chunkCache *chunk_cache.TieredChunkCache + signature int32 + concurrentWriters *util.LimitedConcurrentExecutor + inodeToPath *InodeToPath + fhmap *FileHandleToInode + dhmap *DirectoryHandleToInode + fuseServer *fuse.Server + IsOverQuota bool +} + +func NewSeaweedFileSystem(option *Option) *WFS { + wfs := &WFS{ + RawFileSystem: fuse.NewDefaultRawFileSystem(), + option: option, + signature: util.RandomInt32(), + inodeToPath: NewInodeToPath(util.FullPath(option.FilerMountRootPath)), + fhmap: NewFileHandleToInode(), + dhmap: NewDirectoryHandleToInode(), + } + + wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses)) + wfs.option.setupUniqueCacheDirectory() + if option.CacheSizeMB > 0 { + wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024) + } + + wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), option.UidGidMapper, + util.FullPath(option.FilerMountRootPath), + func(path util.FullPath) { + wfs.inodeToPath.MarkChildrenCached(path) + }, func(path util.FullPath) bool { + return wfs.inodeToPath.IsChildrenCached(path) + }, func(filePath util.FullPath, entry *filer_pb.Entry) { + }) + grace.OnInterrupt(func() { + wfs.metaCache.Shutdown() + os.RemoveAll(option.getUniqueCacheDir()) + }) + + if wfs.option.ConcurrentWriters > 0 { + wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters) + } + return wfs +} + +func (wfs *WFS) StartBackgroundTasks() { + startTime := time.Now() + go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano()) + go wfs.loopCheckQuota() +} + +func (wfs *WFS) String() string { + return "seaweedfs" +} + +func (wfs *WFS) Init(server *fuse.Server) { + wfs.fuseServer = server +} + +func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) { + path, status = wfs.inodeToPath.GetPath(inode) + if status != fuse.OK { + return + } + var found bool + if fh, found = wfs.fhmap.FindFileHandle(inode); found { + entry = fh.GetEntry() + if entry != nil && fh.entry.Attributes == nil { + entry.Attributes = &filer_pb.FuseAttributes{} + } + return path, fh, entry, fuse.OK + } + entry, status = wfs.maybeLoadEntry(path) + return +} + +func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) { + + // glog.V(3).Infof("read entry cache miss %s", fullpath) + dir, name := fullpath.DirAndName() + + // return a valid entry for the mount root + if string(fullpath) == wfs.option.FilerMountRootPath { + return &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: wfs.option.MountMtime.Unix(), + FileMode: uint32(wfs.option.MountMode), + Uid: wfs.option.MountUid, + Gid: wfs.option.MountGid, + Crtime: wfs.option.MountCtime.Unix(), + }, + }, fuse.OK + } + + // read from async meta cache + meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir)) + cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath) + if cacheErr == filer_pb.ErrNotFound { + return nil, fuse.ENOENT + } + return cachedEntry.ToProtoEntry(), fuse.OK +} + +func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { + if wfs.option.VolumeServerAccess == "filerProxy" { + return func(fileId string) (targetUrls []string, err error) { + return []string{"http://" + wfs.getCurrentFiler().ToHttpAddress() + "/?proxyChunkId=" + fileId}, nil + } + } + return filer.LookupFn(wfs) +} + +func (wfs *WFS) getCurrentFiler() pb.ServerAddress { + return wfs.option.FilerAddresses[wfs.option.filerIndex] +} + +func (option *Option) setupUniqueCacheDirectory() { + cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8] + option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId) + option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "swap") + os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask) +} + +func (option *Option) getTempFilePageDir() string { + return option.uniqueCacheTempPageDir +} + +func (option *Option) getUniqueCacheDir() string { + return option.uniqueCacheDir +} diff --git a/weed/mount/weedfs_attr.go b/weed/mount/weedfs_attr.go new file mode 100644 index 000000000..be504f5e2 --- /dev/null +++ b/weed/mount/weedfs_attr.go @@ -0,0 +1,241 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "syscall" + "time" +) + +func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) { + if input.NodeId == 1 { + wfs.setRootAttr(out) + return fuse.OK + } + + _, _, entry, status := wfs.maybeReadEntry(input.NodeId) + if status == fuse.OK { + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry) + return status + } else { + if fh, found := wfs.fhmap.FindFileHandle(input.NodeId); found { + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, input.NodeId, fh.entry) + out.Nlink = 0 + return fuse.OK + } + } + + return status +} + +func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) { + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + path, fh, entry, status := wfs.maybeReadEntry(input.NodeId) + if status != fuse.OK { + return status + } + if fh != nil { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + } + + if size, ok := input.GetSize(); ok { + glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks)) + if size < filer.FileSize(entry) { + // fmt.Printf("truncate %v \n", fullPath) + var chunks []*filer_pb.FileChunk + var truncatedChunks []*filer_pb.FileChunk + for _, chunk := range entry.Chunks { + int64Size := int64(chunk.Size) + if chunk.Offset+int64Size > int64(size) { + // this chunk is truncated + int64Size = int64(size) - chunk.Offset + if int64Size > 0 { + chunks = append(chunks, chunk) + glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size) + chunk.Size = uint64(int64Size) + } else { + glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString()) + truncatedChunks = append(truncatedChunks, chunk) + } + } + } + // set the new chunks and reset entry cache + entry.Chunks = chunks + if fh != nil { + fh.entryViewCache = nil + } + } + entry.Attributes.Mtime = time.Now().Unix() + entry.Attributes.FileSize = size + + } + + if mode, ok := input.GetMode(); ok { + // glog.V(4).Infof("setAttr mode %o", mode) + entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode) + if input.NodeId == 1 { + wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode)) + } + } + + if uid, ok := input.GetUID(); ok { + entry.Attributes.Uid = uid + if input.NodeId == 1 { + wfs.option.MountUid = uid + } + } + + if gid, ok := input.GetGID(); ok { + entry.Attributes.Gid = gid + if input.NodeId == 1 { + wfs.option.MountGid = gid + } + } + + if atime, ok := input.GetATime(); ok { + entry.Attributes.Mtime = atime.Unix() + } + + if mtime, ok := input.GetMTime(); ok { + entry.Attributes.Mtime = mtime.Unix() + } + + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry) + + if fh != nil { + fh.dirtyMetadata = true + return fuse.OK + } + + return wfs.saveEntry(path, entry) + +} + +func (wfs *WFS) setRootAttr(out *fuse.AttrOut) { + now := uint64(time.Now().Unix()) + out.AttrValid = 119 + out.Ino = 1 + setBlksize(&out.Attr, blockSize) + out.Uid = wfs.option.MountUid + out.Gid = wfs.option.MountGid + out.Mtime = now + out.Ctime = now + out.Atime = now + out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode) + out.Nlink = 1 +} + +func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) { + out.Ino = inode + out.Size = filer.FileSize(entry) + out.Blocks = (out.Size + blockSize - 1) / blockSize + setBlksize(out, blockSize) + out.Mtime = uint64(entry.Attributes.Mtime) + out.Ctime = uint64(entry.Attributes.Mtime) + out.Atime = uint64(entry.Attributes.Mtime) + out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode)) + if entry.HardLinkCounter > 0 { + out.Nlink = uint32(entry.HardLinkCounter) + } else { + out.Nlink = 1 + } + out.Uid = entry.Attributes.Uid + out.Gid = entry.Attributes.Gid + out.Rdev = entry.Attributes.Rdev +} + +func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) { + out.Ino = inode + out.Size = entry.FileSize + out.Blocks = (out.Size + blockSize - 1) / blockSize + setBlksize(out, blockSize) + out.Atime = uint64(entry.Attr.Mtime.Unix()) + out.Mtime = uint64(entry.Attr.Mtime.Unix()) + out.Ctime = uint64(entry.Attr.Mtime.Unix()) + out.Mode = toSyscallMode(entry.Attr.Mode) + if entry.HardLinkCounter > 0 { + out.Nlink = uint32(entry.HardLinkCounter) + } else { + out.Nlink = 1 + } + out.Uid = entry.Attr.Uid + out.Gid = entry.Attr.Gid + out.Rdev = entry.Attr.Rdev +} + +func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) { + out.NodeId = inode + out.Generation = 1 + out.EntryValid = 1 + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, inode, entry) +} + +func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) { + out.NodeId = inode + out.Generation = 1 + out.EntryValid = 1 + out.AttrValid = 1 + wfs.setAttrByFilerEntry(&out.Attr, inode, entry) +} + +func chmod(existing uint32, mode uint32) uint32 { + return existing&^07777 | mode&07777 +} + +func toSyscallMode(mode os.FileMode) uint32 { + return toSyscallType(mode) | uint32(mode) +} + +func toSyscallType(mode os.FileMode) uint32 { + switch mode & os.ModeType { + case os.ModeDir: + return syscall.S_IFDIR + case os.ModeSymlink: + return syscall.S_IFLNK + case os.ModeNamedPipe: + return syscall.S_IFIFO + case os.ModeSocket: + return syscall.S_IFSOCK + case os.ModeDevice: + return syscall.S_IFBLK + case os.ModeCharDevice: + return syscall.S_IFCHR + default: + return syscall.S_IFREG + } +} + +func toOsFileType(mode uint32) os.FileMode { + switch mode & (syscall.S_IFMT & 0xffff) { + case syscall.S_IFDIR: + return os.ModeDir + case syscall.S_IFLNK: + return os.ModeSymlink + case syscall.S_IFIFO: + return os.ModeNamedPipe + case syscall.S_IFSOCK: + return os.ModeSocket + case syscall.S_IFBLK: + return os.ModeDevice + case syscall.S_IFCHR: + return os.ModeCharDevice + default: + return 0 + } +} + +func toOsFileMode(mode uint32) os.FileMode { + return toOsFileType(mode) | os.FileMode(mode&07777) +} diff --git a/weed/mount/weedfs_attr_darwin.go b/weed/mount/weedfs_attr_darwin.go new file mode 100644 index 000000000..e7767d4a6 --- /dev/null +++ b/weed/mount/weedfs_attr_darwin.go @@ -0,0 +1,8 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +func setBlksize(out *fuse.Attr, size uint32) { +} diff --git a/weed/mount/weedfs_attr_linux.go b/weed/mount/weedfs_attr_linux.go new file mode 100644 index 000000000..56be62e62 --- /dev/null +++ b/weed/mount/weedfs_attr_linux.go @@ -0,0 +1,9 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +func setBlksize(out *fuse.Attr, size uint32) { + out.Blksize = size +} diff --git a/weed/mount/weedfs_dir_lookup.go b/weed/mount/weedfs_dir_lookup.go new file mode 100644 index 000000000..fdc16aad5 --- /dev/null +++ b/weed/mount/weedfs_dir_lookup.go @@ -0,0 +1,67 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" +) + +// Lookup is called by the kernel when the VFS wants to know +// about a file inside a directory. Many lookup calls can +// occur in parallel, but only one call happens for each (dir, +// name) pair. + +func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + dirPath, code := wfs.inodeToPath.GetPath(header.NodeId) + if code != fuse.OK { + return + } + + fullFilePath := dirPath.Child(name) + + visitErr := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath) + if visitErr != nil { + glog.Errorf("dir Lookup %s: %v", dirPath, visitErr) + return fuse.EIO + } + localEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullFilePath) + if cacheErr == filer_pb.ErrNotFound { + return fuse.ENOENT + } + + if localEntry == nil { + // glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath) + entry, err := filer_pb.GetEntry(wfs, fullFilePath) + if err != nil { + glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err) + return fuse.ENOENT + } + localEntry = filer.FromPbEntry(string(dirPath), entry) + } else { + glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath) + } + + if localEntry == nil { + return fuse.ENOENT + } + + inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.Crtime.Unix(), localEntry.IsDirectory(), len(localEntry.HardLinkId) > 0, localEntry.Inode, true) + + if fh, found := wfs.fhmap.FindFileHandle(inode); found && fh.entry != nil { + glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(fh.entry)) + localEntry = filer.FromPbEntry(string(dirPath), fh.entry) + } + + wfs.outputFilerEntry(out, inode, localEntry) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_dir_mkrm.go b/weed/mount/weedfs_dir_mkrm.go new file mode 100644 index 000000000..4246f0a4c --- /dev/null +++ b/weed/mount/weedfs_dir_mkrm.go @@ -0,0 +1,121 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "strings" + "syscall" + "time" +) + +/** Create a directory + * + * Note that the mode argument may not have the type specification + * bits set, i.e. S_ISDIR(mode) can be false. To obtain the + * correct directory type bits use mode|S_IFDIR + * */ +func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + if s := checkName(name); s != fuse.OK { + return s + } + + newEntry := &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask), + Uid: in.Uid, + Gid: in.Gid, + }, + } + + dirFullPath, code := wfs.inodeToPath.GetPath(in.NodeId) + if code != fuse.OK { + return + } + + entryFullPath := dirFullPath.Child(name) + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(newEntry) + defer wfs.mapPbIdFromFilerToLocal(newEntry) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirFullPath), + Entry: newEntry, + Signatures: []int32{wfs.signature}, + } + + glog.V(1).Infof("mkdir: %v", request) + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.V(0).Infof("mkdir %s: %v", entryFullPath, err) + return err + } + + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("local mkdir dir %s: %v", entryFullPath, err) + } + + return nil + }) + + glog.V(3).Infof("mkdir %s: %v", entryFullPath, err) + + if err != nil { + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, true, false, 0, true) + + wfs.outputPbEntry(out, inode, newEntry) + + return fuse.OK + +} + +/** Remove a directory */ +func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) { + + if name == "." { + return fuse.Status(syscall.EINVAL) + } + if name == ".." { + return fuse.Status(syscall.ENOTEMPTY) + } + + dirFullPath, code := wfs.inodeToPath.GetPath(header.NodeId) + if code != fuse.OK { + return + } + entryFullPath := dirFullPath.Child(name) + + glog.V(3).Infof("remove directory: %v", entryFullPath) + ignoreRecursiveErr := true // ignore recursion error since the OS should manage it + err := filer_pb.Remove(wfs, string(dirFullPath), name, true, false, ignoreRecursiveErr, false, []int32{wfs.signature}) + if err != nil { + glog.V(0).Infof("remove %s: %v", entryFullPath, err) + if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) { + return fuse.Status(syscall.ENOTEMPTY) + } + return fuse.ENOENT + } + + wfs.metaCache.DeleteEntry(context.Background(), entryFullPath) + wfs.inodeToPath.RemovePath(entryFullPath) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_dir_read.go b/weed/mount/weedfs_dir_read.go new file mode 100644 index 000000000..dd34e1ab5 --- /dev/null +++ b/weed/mount/weedfs_dir_read.go @@ -0,0 +1,240 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/hanwen/go-fuse/v2/fuse" + "math" + "sync" +) + +type DirectoryHandleId uint64 + +const ( + directoryStreamBaseOffset = 2 // . & .. +) + +type DirectoryHandle struct { + isFinished bool + entryStream []*filer.Entry + entryStreamOffset uint64 +} + +func (dh *DirectoryHandle) reset() { + *dh = DirectoryHandle{ + isFinished: false, + entryStream: []*filer.Entry{}, + entryStreamOffset: directoryStreamBaseOffset, + } +} + +type DirectoryHandleToInode struct { + // shares the file handle id sequencer with FileHandleToInode{nextFh} + sync.Mutex + dir2inode map[DirectoryHandleId]*DirectoryHandle +} + +func NewDirectoryHandleToInode() *DirectoryHandleToInode { + return &DirectoryHandleToInode{ + dir2inode: make(map[DirectoryHandleId]*DirectoryHandle), + } +} + +func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) { + wfs.fhmap.Lock() + fh := wfs.fhmap.nextFh + wfs.fhmap.nextFh++ + wfs.fhmap.Unlock() + + wfs.dhmap.Lock() + defer wfs.dhmap.Unlock() + dh := new(DirectoryHandle) + dh.reset() + wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh + return DirectoryHandleId(fh), dh +} + +func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle { + wfs.dhmap.Lock() + defer wfs.dhmap.Unlock() + if dh, found := wfs.dhmap.dir2inode[dhid]; found { + return dh + } + dh := new(DirectoryHandle) + dh.reset() + wfs.dhmap.dir2inode[dhid] = dh + return dh +} + +func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) { + wfs.dhmap.Lock() + defer wfs.dhmap.Unlock() + delete(wfs.dhmap.dir2inode, dhid) +} + +// Directory handling + +/** Open directory + * + * Unless the 'default_permissions' mount option is given, + * this method should check if opendir is permitted for this + * directory. Optionally opendir may also return an arbitrary + * filehandle in the fuse_file_info structure, which will be + * passed to readdir, releasedir and fsyncdir. + */ +func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) { + if !wfs.inodeToPath.HasInode(input.NodeId) { + return fuse.ENOENT + } + dhid, _ := wfs.AcquireDirectoryHandle() + out.Fh = uint64(dhid) + return fuse.OK +} + +/** Release directory + * + * If the directory has been removed after the call to opendir, the + * path parameter will be NULL. + */ +func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) { + wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh)) +} + +/** Synchronize directory contents + * + * If the directory has been removed after the call to opendir, the + * path parameter will be NULL. + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data + */ +func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) { + return fuse.OK +} + +/** Read directory + * + * The filesystem may choose between two modes of operation: + * + * 1) The readdir implementation ignores the offset parameter, and + * passes zero to the filler function's offset. The filler + * function will not return '1' (unless an error happens), so the + * whole directory is read in a single readdir operation. + * + * 2) The readdir implementation keeps track of the offsets of the + * directory entries. It uses the offset parameter and always + * passes non-zero offset to the filler function. When the buffer + * is full (or an error happens) the filler function will return + * '1'. + */ +func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) { + return wfs.doReadDirectory(input, out, false) +} + +func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) { + return wfs.doReadDirectory(input, out, true) +} + +func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status { + dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh)) + if input.Offset == 0 { + dh.reset() + } else if dh.isFinished && input.Offset >= dh.entryStreamOffset { + entryCurrentIndex := input.Offset - dh.entryStreamOffset + if uint64(len(dh.entryStream)) <= entryCurrentIndex { + return fuse.OK + } + } + + isEarlyTerminated := false + dirPath, code := wfs.inodeToPath.GetPath(input.NodeId) + if code != fuse.OK { + return code + } + + var dirEntry fuse.DirEntry + processEachEntryFn := func(entry *filer.Entry) bool { + dirEntry.Name = entry.Name() + dirEntry.Mode = toSyscallMode(entry.Mode) + inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, isPlusMode) + dirEntry.Ino = inode + if !isPlusMode { + if !out.AddDirEntry(dirEntry) { + isEarlyTerminated = true + return false + } + } else { + entryOut := out.AddDirLookupEntry(dirEntry) + if entryOut == nil { + isEarlyTerminated = true + return false + } + if fh, found := wfs.fhmap.FindFileHandle(inode); found { + glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name)) + entry = filer.FromPbEntry(string(dirPath), fh.entry) + } + wfs.outputFilerEntry(entryOut, inode, entry) + } + return true + } + + if input.Offset < directoryStreamBaseOffset { + if !isPlusMode { + if input.Offset == 0 { + out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."}) + } + out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."}) + } else { + if input.Offset == 0 { + out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."}) + } + out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."}) + } + input.Offset = directoryStreamBaseOffset + } + + var lastEntryName string + if input.Offset >= dh.entryStreamOffset { + if input.Offset > dh.entryStreamOffset { + entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1 + if uint64(len(dh.entryStream)) > entryPreviousIndex { + lastEntryName = dh.entryStream[entryPreviousIndex].Name() + dh.entryStream = dh.entryStream[entryPreviousIndex:] + dh.entryStreamOffset = input.Offset - 1 + } + } + entryCurrentIndex := input.Offset - dh.entryStreamOffset + for uint64(len(dh.entryStream)) > entryCurrentIndex { + entry := dh.entryStream[entryCurrentIndex] + if processEachEntryFn(entry) { + lastEntryName = entry.Name() + entryCurrentIndex++ + } else { + // early terminated + return fuse.OK + } + } + } + + var err error + if err = meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil { + glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) + return fuse.EIO + } + listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool { + dh.entryStream = append(dh.entryStream, entry) + return processEachEntryFn(entry) + }) + if listErr != nil { + glog.Errorf("list meta cache: %v", listErr) + return fuse.EIO + } + + if !isEarlyTerminated { + dh.isFinished = true + } + + return fuse.OK +} diff --git a/weed/mount/weedfs_file_io.go b/weed/mount/weedfs_file_io.go new file mode 100644 index 000000000..7039b14ec --- /dev/null +++ b/weed/mount/weedfs_file_io.go @@ -0,0 +1,100 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +/** + * Open a file + * + * Open flags are available in fi->flags. The following rules + * apply. + * + * - Creation (O_CREAT, O_EXCL, O_NOCTTY) flags will be + * filtered out / handled by the kernel. + * + * - Access modes (O_RDONLY, O_WRONLY, O_RDWR) should be used + * by the filesystem to check if the operation is + * permitted. If the ``-o default_permissions`` mount + * option is given, this check is already done by the + * kernel before calling open() and may thus be omitted by + * the filesystem. + * + * - When writeback caching is enabled, the kernel may send + * read requests even for files opened with O_WRONLY. The + * filesystem should be prepared to handle this. + * + * - When writeback caching is disabled, the filesystem is + * expected to properly handle the O_APPEND flag and ensure + * that each write is appending to the end of the file. + * + * - When writeback caching is enabled, the kernel will + * handle O_APPEND. However, unless all changes to the file + * come through the kernel this will not work reliably. The + * filesystem should thus either ignore the O_APPEND flag + * (and let the kernel handle it), or return an error + * (indicating that reliably O_APPEND is not available). + * + * Filesystem may store an arbitrary file handle (pointer, + * index, etc) in fi->fh, and use this in other all other file + * operations (read, write, flush, release, fsync). + * + * Filesystem may also implement stateless file I/O and not store + * anything in fi->fh. + * + * There are also some flags (direct_io, keep_cache) which the + * filesystem may set in fi, to change the way the file is opened. + * See fuse_file_info structure in for more details. + * + * If this request is answered with an error code of ENOSYS + * and FUSE_CAP_NO_OPEN_SUPPORT is set in + * `fuse_conn_info.capable`, this is treated as success and + * future calls to open and release will also succeed without being + * sent to the filesystem process. + * + * Valid replies: + * fuse_reply_open + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information +*/ +func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) { + var fileHandle *FileHandle + fileHandle, status = wfs.AcquireHandle(in.NodeId, in.Uid, in.Gid) + if status == fuse.OK { + out.Fh = uint64(fileHandle.fh) + // TODO https://github.com/libfuse/libfuse/blob/master/include/fuse_common.h#L64 + } + return status +} + +/** + * Release an open file + * + * Release is called when there are no more references to an open + * file: all file descriptors are closed and all memory mappings + * are unmapped. + * + * For every open call there will be exactly one release call (unless + * the filesystem is force-unmounted). + * + * The filesystem may reply with an error, but error values are + * not returned to close() or munmap() which triggered the + * release. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * fi->flags will contain the same flags as for open. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information + */ +func (wfs *WFS) Release(cancel <-chan struct{}, in *fuse.ReleaseIn) { + wfs.ReleaseHandle(FileHandleId(in.Fh)) +} diff --git a/weed/mount/weedfs_file_mkrm.go b/weed/mount/weedfs_file_mkrm.go new file mode 100644 index 000000000..86d4c4d41 --- /dev/null +++ b/weed/mount/weedfs_file_mkrm.go @@ -0,0 +1,152 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "syscall" + "time" +) + +/** + * Create and open a file + * + * If the file does not exist, first create it with the specified + * mode, and then open it. + * + * If this method is not implemented or under Linux kernel + * versions earlier than 2.6.15, the mknod() and open() methods + * will be called instead. + */ +func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, out *fuse.CreateOut) (code fuse.Status) { + // if implemented, need to use + // inode := wfs.inodeToPath.Lookup(entryFullPath) + // to ensure nlookup counter + return fuse.ENOSYS +} + +/** Create a file node + * + * This is called for creation of all non-directory, non-symlink + * nodes. If the filesystem defines a create() method, then for + * regular files that will be called instead. + */ +func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + if s := checkName(name); s != fuse.OK { + return s + } + + dirFullPath, code := wfs.inodeToPath.GetPath(in.NodeId) + if code != fuse.OK { + return + } + + entryFullPath := dirFullPath.Child(name) + fileMode := toOsFileMode(in.Mode) + now := time.Now().Unix() + inode := wfs.inodeToPath.AllocateInode(entryFullPath, now) + + newEntry := &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: now, + Crtime: now, + FileMode: uint32(fileMode), + Uid: in.Uid, + Gid: in.Gid, + TtlSec: wfs.option.TtlSec, + Rdev: in.Rdev, + Inode: inode, + }, + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(newEntry) + defer wfs.mapPbIdFromFilerToLocal(newEntry) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirFullPath), + Entry: newEntry, + Signatures: []int32{wfs.signature}, + SkipCheckParentDirectory: true, + } + + glog.V(1).Infof("mknod: %v", request) + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.V(0).Infof("mknod %s: %v", entryFullPath, err) + return err + } + + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("local mknod %s: %v", entryFullPath, err) + } + + return nil + }) + + glog.V(3).Infof("mknod %s: %v", entryFullPath, err) + + if err != nil { + return fuse.EIO + } + + // this is to increase nlookup counter + inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, false, inode, true) + + wfs.outputPbEntry(out, inode, newEntry) + + return fuse.OK + +} + +/** Remove a file */ +func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) { + + dirFullPath, code := wfs.inodeToPath.GetPath(header.NodeId) + if code != fuse.OK { + if code == fuse.ENOENT { + return fuse.OK + } + return code + } + entryFullPath := dirFullPath.Child(name) + + entry, code := wfs.maybeLoadEntry(entryFullPath) + if code != fuse.OK { + if code == fuse.ENOENT { + return fuse.OK + } + return code + } + + // first, ensure the filer store can correctly delete + glog.V(3).Infof("remove file: %v", entryFullPath) + isDeleteData := entry != nil && entry.HardLinkCounter <= 1 + err := filer_pb.Remove(wfs, string(dirFullPath), name, isDeleteData, false, false, false, []int32{wfs.signature}) + if err != nil { + glog.V(0).Infof("remove %s: %v", entryFullPath, err) + return fuse.OK + } + + // then, delete meta cache + if err = wfs.metaCache.DeleteEntry(context.Background(), entryFullPath); err != nil { + glog.V(3).Infof("local DeleteEntry %s: %v", entryFullPath, err) + return fuse.EIO + } + + wfs.metaCache.DeleteEntry(context.Background(), entryFullPath) + wfs.inodeToPath.RemovePath(entryFullPath) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_file_read.go b/weed/mount/weedfs_file_read.go new file mode 100644 index 000000000..00143a5b4 --- /dev/null +++ b/weed/mount/weedfs_file_read.go @@ -0,0 +1,59 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/hanwen/go-fuse/v2/fuse" + "io" +) + +/** + * Read data + * + * Read should send exactly the number of bytes requested except + * on EOF or error, otherwise the rest of the data will be + * substituted with zeroes. An exception to this is when the file + * has been opened in 'direct_io' mode, in which case the return + * value of the read system call will reflect the return value of + * this operation. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * Valid replies: + * fuse_reply_buf + * fuse_reply_iov + * fuse_reply_data + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param size number of bytes to read + * @param off offset to read from + * @param fi file information + */ +func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse.ReadResult, fuse.Status) { + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return nil, fuse.ENOENT + } + + offset := int64(in.Offset) + fh.lockForRead(offset, len(buff)) + defer fh.unlockForRead(offset, len(buff)) + + totalRead, err := fh.readFromChunks(buff, offset) + if err == nil || err == io.EOF { + maxStop := fh.readFromDirtyPages(buff, offset) + totalRead = max(maxStop-offset, totalRead) + } + if err == io.EOF { + err = nil + } + + if err != nil { + glog.Warningf("file handle read %s %d: %v", fh.FullPath(), totalRead, err) + return nil, fuse.EIO + } + + return fuse.ReadResultData(buff[:totalRead]), fuse.OK +} diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go new file mode 100644 index 000000000..b7fffaaa3 --- /dev/null +++ b/weed/mount/weedfs_file_sync.go @@ -0,0 +1,189 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "syscall" + "time" +) + +/** + * Flush method + * + * This is called on each close() of the opened file. + * + * Since file descriptors can be duplicated (dup, dup2, fork), for + * one open call there may be many flush calls. + * + * Filesystems shouldn't assume that flush will always be called + * after some writes, or that if will be called at all. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * NOTE: the name of the method is misleading, since (unlike + * fsync) the filesystem is not forced to flush pending writes. + * One reason to flush data is if the filesystem wants to return + * write errors during close. However, such use is non-portable + * because POSIX does not require [close] to wait for delayed I/O to + * complete. + * + * If the filesystem supports file locking operations (setlk, + * getlk) it should remove all locks belonging to 'fi->owner'. + * + * If this request is answered with an error code of ENOSYS, + * this is treated as success and future calls to flush() will + * succeed automatically without being send to the filesystem + * process. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information + * + * [close]: http://pubs.opengroup.org/onlinepubs/9699919799/functions/close.html + */ +func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status { + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return fuse.ENOENT + } + + fh.Lock() + defer fh.Unlock() + + return wfs.doFlush(fh, in.Uid, in.Gid) +} + +/** + * Synchronize file contents + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data. + * + * If this request is answered with an error code of ENOSYS, + * this is treated as success and future calls to fsync() will + * succeed automatically without being send to the filesystem + * process. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param datasync flag indicating if only data should be flushed + * @param fi file information + */ +func (wfs *WFS) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Status) { + + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return fuse.ENOENT + } + + fh.Lock() + defer fh.Unlock() + + return wfs.doFlush(fh, in.Uid, in.Gid) + +} + +func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status { + // flush works at fh level + fileFullPath := fh.FullPath() + dir, name := fileFullPath.DirAndName() + // send the data to the OS + glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.handle) + + if !wfs.IsOverQuota { + if err := fh.dirtyPages.FlushData(); err != nil { + glog.Errorf("%v doFlush: %v", fileFullPath, err) + return fuse.EIO + } + } + + if !fh.dirtyMetadata { + return fuse.OK + } + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + + entry := fh.entry + if entry == nil { + return nil + } + entry.Name = name // this flush may be just after a rename operation + + if entry.Attributes != nil { + entry.Attributes.Mime = fh.contentType + if entry.Attributes.Uid == 0 { + entry.Attributes.Uid = uid + } + if entry.Attributes.Gid == 0 { + entry.Attributes.Gid = gid + } + if entry.Attributes.Crtime == 0 { + entry.Attributes.Crtime = time.Now().Unix() + } + entry.Attributes.Mtime = time.Now().Unix() + } + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dir), + Entry: entry, + Signatures: []int32{wfs.signature}, + SkipCheckParentDirectory: true, + } + + glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks)) + for i, chunk := range entry.Chunks { + glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + + manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks) + + chunks, _ := filer.CompactFileChunks(wfs.LookupFn(), nonManifestChunks) + chunks, manifestErr := filer.MaybeManifestize(wfs.saveDataAsChunk(fileFullPath), chunks) + if manifestErr != nil { + // not good, but should be ok + glog.V(0).Infof("MaybeManifestize: %v", manifestErr) + } + entry.Chunks = append(chunks, manifestChunks...) + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.Errorf("fh flush create %s: %v", fileFullPath, err) + return fmt.Errorf("fh flush create %s: %v", fileFullPath, err) + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + + if err == nil { + fh.dirtyMetadata = false + } + + if err != nil { + glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.handle, err) + return fuse.EIO + } + + return fuse.OK +} diff --git a/weed/mount/weedfs_file_write.go b/weed/mount/weedfs_file_write.go new file mode 100644 index 000000000..d14680752 --- /dev/null +++ b/weed/mount/weedfs_file_write.go @@ -0,0 +1,73 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" + "net/http" + "syscall" +) + +/** + * Write data + * + * Write should return exactly the number of bytes requested + * except on error. An exception to this is when the file has + * been opened in 'direct_io' mode, in which case the return value + * of the write system call will reflect the return value of this + * operation. + * + * Unless FUSE_CAP_HANDLE_KILLPRIV is disabled, this method is + * expected to reset the setuid and setgid bits. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * Valid replies: + * fuse_reply_write + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param buf data to write + * @param size number of bytes to write + * @param off offset to write to + * @param fi file information + */ +func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (written uint32, code fuse.Status) { + + if wfs.IsOverQuota { + return 0, fuse.Status(syscall.ENOSPC) + } + + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return 0, fuse.ENOENT + } + + fh.dirtyPages.writerPattern.MonitorWriteAt(int64(in.Offset), int(in.Size)) + + fh.Lock() + defer fh.Unlock() + + entry := fh.entry + if entry == nil { + return 0, fuse.OK + } + + entry.Content = nil + offset := int64(in.Offset) + entry.Attributes.FileSize = uint64(max(offset+int64(len(data)), int64(entry.Attributes.FileSize))) + // glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data)) + + fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsStreamingMode()) + + written = uint32(len(data)) + + if offset == 0 { + // detect mime type + fh.contentType = http.DetectContentType(data) + } + + fh.dirtyMetadata = true + + return written, fuse.OK +} diff --git a/weed/mount/weedfs_filehandle.go b/weed/mount/weedfs_filehandle.go new file mode 100644 index 000000000..d769e51c5 --- /dev/null +++ b/weed/mount/weedfs_filehandle.go @@ -0,0 +1,24 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" +) + +func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) { + var entry *filer_pb.Entry + _, _, entry, status = wfs.maybeReadEntry(inode) + if status == fuse.OK { + // need to AcquireFileHandle again to ensure correct handle counter + fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry) + } + return +} + +func (wfs *WFS) ReleaseHandle(handleId FileHandleId) { + wfs.fhmap.ReleaseByHandle(handleId) +} + +func (wfs *WFS) GetHandle(handleId FileHandleId) *FileHandle { + return wfs.fhmap.GetFileHandle(handleId) +} diff --git a/weed/mount/weedfs_forget.go b/weed/mount/weedfs_forget.go new file mode 100644 index 000000000..4212adeb6 --- /dev/null +++ b/weed/mount/weedfs_forget.go @@ -0,0 +1,69 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/util" +) + +// Forget is called when the kernel discards entries from its +// dentry cache. This happens on unmount, and when the kernel +// is short on memory. Since it is not guaranteed to occur at +// any moment, and since there is no return value, Forget +// should not do I/O, as there is no channel to report back +// I/O errors. +// from https://github.com/libfuse/libfuse/blob/master/include/fuse_lowlevel.h +/** + * Forget about an inode + * + * This function is called when the kernel removes an inode + * from its internal caches. + * + * The inode's lookup count increases by one for every call to + * fuse_reply_entry and fuse_reply_create. The nlookup parameter + * indicates by how much the lookup count should be decreased. + * + * Inodes with a non-zero lookup count may receive request from + * the kernel even after calls to unlink, rmdir or (when + * overwriting an existing file) rename. Filesystems must handle + * such requests properly and it is recommended to defer removal + * of the inode until the lookup count reaches zero. Calls to + * unlink, rmdir or rename will be followed closely by forget + * unless the file or directory is open, in which case the + * kernel issues forget only after the release or releasedir + * calls. + * + * Note that if a file system will be exported over NFS the + * inodes lifetime must extend even beyond forget. See the + * generation field in struct fuse_entry_param above. + * + * On unmount the lookup count for all inodes implicitly drops + * to zero. It is not guaranteed that the file system will + * receive corresponding forget messages for the affected + * inodes. + * + * Valid replies: + * fuse_reply_none + * + * @param req request handle + * @param ino the inode number + * @param nlookup the number of lookups to forget + */ +/* +https://libfuse.github.io/doxygen/include_2fuse__lowlevel_8h.html + +int fuse_reply_entry ( fuse_req_t req, +const struct fuse_entry_param * e +) +Reply with a directory entry + +Possible requests: lookup, mknod, mkdir, symlink, link + +Side effects: increments the lookup count on success + +*/ +func (wfs *WFS) Forget(nodeid, nlookup uint64) { + wfs.inodeToPath.Forget(nodeid, nlookup, func(dir util.FullPath) { + wfs.metaCache.DeleteFolderChildren(context.Background(), dir) + }) + wfs.fhmap.ReleaseByInode(nodeid) +} diff --git a/weed/mount/weedfs_grpc_server.go b/weed/mount/weedfs_grpc_server.go new file mode 100644 index 000000000..4b2fdffa6 --- /dev/null +++ b/weed/mount/weedfs_grpc_server.go @@ -0,0 +1,17 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/mount_pb" +) + +func (wfs *WFS) Configure(ctx context.Context, request *mount_pb.ConfigureRequest) (*mount_pb.ConfigureResponse, error) { + if wfs.option.Collection == "" { + return nil, fmt.Errorf("mount quota only works when mounted to a new folder with a collection") + } + glog.V(0).Infof("quota changed from %d to %d", wfs.option.Quota, request.CollectionCapacity) + wfs.option.Quota = request.GetCollectionCapacity() + return &mount_pb.ConfigureResponse{}, nil +} diff --git a/weed/mount/weedfs_link.go b/weed/mount/weedfs_link.go new file mode 100644 index 000000000..2ab412fd5 --- /dev/null +++ b/weed/mount/weedfs_link.go @@ -0,0 +1,110 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "syscall" + "time" +) + +/* +What is an inode? +If the file is an hardlinked file: + use the hardlink id as inode +Otherwise: + use the file path as inode + +When creating a link: + use the original file inode +*/ + +/** Create a hard link to a file */ +func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + if s := checkName(name); s != fuse.OK { + return s + } + + newParentPath, code := wfs.inodeToPath.GetPath(in.NodeId) + if code != fuse.OK { + return + } + oldEntryPath, code := wfs.inodeToPath.GetPath(in.Oldnodeid) + if code != fuse.OK { + return + } + oldParentPath, _ := oldEntryPath.DirAndName() + + oldEntry, status := wfs.maybeLoadEntry(oldEntryPath) + if status != fuse.OK { + return status + } + + // update old file to hardlink mode + if len(oldEntry.HardLinkId) == 0 { + oldEntry.HardLinkId = filer.NewHardLinkId() + oldEntry.HardLinkCounter = 1 + } + oldEntry.HardLinkCounter++ + updateOldEntryRequest := &filer_pb.UpdateEntryRequest{ + Directory: oldParentPath, + Entry: oldEntry, + Signatures: []int32{wfs.signature}, + } + + // CreateLink 1.2 : update new file to hardlink mode + oldEntry.Attributes.Mtime = time.Now().Unix() + request := &filer_pb.CreateEntryRequest{ + Directory: string(newParentPath), + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: oldEntry.Attributes, + Chunks: oldEntry.Chunks, + Extended: oldEntry.Extended, + HardLinkId: oldEntry.HardLinkId, + HardLinkCounter: oldEntry.HardLinkCounter, + }, + Signatures: []int32{wfs.signature}, + } + + // apply changes to the filer, and also apply to local metaCache + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.UpdateEntry(client, updateOldEntryRequest); err != nil { + return err + } + wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry)) + + if err := filer_pb.CreateEntry(client, request); err != nil { + return err + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + + newEntryPath := newParentPath.Child(name) + + if err != nil { + glog.V(0).Infof("Link %v -> %s: %v", oldEntryPath, newEntryPath, err) + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(newEntryPath, oldEntry.Attributes.Crtime, oldEntry.IsDirectory, true, oldEntry.Attributes.Inode, true) + + wfs.outputPbEntry(out, inode, request.Entry) + + return fuse.OK +} diff --git a/weed/mount/weedfs_quota.go b/weed/mount/weedfs_quota.go new file mode 100644 index 000000000..ac3e58e62 --- /dev/null +++ b/weed/mount/weedfs_quota.go @@ -0,0 +1,54 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" +) + +func (wfs *WFS) loopCheckQuota() { + + for { + + time.Sleep(61 * time.Second) + + if wfs.option.Quota <= 0 { + continue + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.StatisticsRequest{ + Collection: wfs.option.Collection, + Replication: wfs.option.Replication, + Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), + DiskType: string(wfs.option.DiskType), + } + + resp, err := client.Statistics(context.Background(), request) + if err != nil { + glog.V(0).Infof("reading quota usage %v: %v", request, err) + return err + } + glog.V(4).Infof("read quota usage: %+v", resp) + + isOverQuota := int64(resp.UsedSize) > wfs.option.Quota + if isOverQuota && !wfs.IsOverQuota { + glog.Warningf("Quota Exceeded! quota:%d used:%d", wfs.option.Quota, resp.UsedSize) + } else if !isOverQuota && wfs.IsOverQuota { + glog.Warningf("Within quota limit! quota:%d used:%d", wfs.option.Quota, resp.UsedSize) + } + wfs.IsOverQuota = isOverQuota + + return nil + }) + + if err != nil { + glog.Warningf("read quota usage: %v", err) + } + + } + +} diff --git a/weed/mount/weedfs_rename.go b/weed/mount/weedfs_rename.go new file mode 100644 index 000000000..23caa48cd --- /dev/null +++ b/weed/mount/weedfs_rename.go @@ -0,0 +1,251 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" + "io" + "strings" + "syscall" +) + +/** Rename a file + * + * If the target exists it should be atomically replaced. If + * the target's inode's lookup count is non-zero, the file + * system is expected to postpone any removal of the inode + * until the lookup count reaches zero (see description of the + * forget function). + * + * If this request is answered with an error code of ENOSYS, this is + * treated as a permanent failure with error code EINVAL, i.e. all + * future bmap requests will fail with EINVAL without being + * send to the filesystem process. + * + * *flags* may be `RENAME_EXCHANGE` or `RENAME_NOREPLACE`. If + * RENAME_NOREPLACE is specified, the filesystem must not + * overwrite *newname* if it exists and return an error + * instead. If `RENAME_EXCHANGE` is specified, the filesystem + * must atomically exchange the two files, i.e. both must + * exist and neither may be deleted. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param parent inode number of the old parent directory + * @param name old name + * @param newparent inode number of the new parent directory + * @param newname new name + */ +/* +renameat2() + renameat2() has an additional flags argument. A renameat2() call + with a zero flags argument is equivalent to renameat(). + + The flags argument is a bit mask consisting of zero or more of + the following flags: + + RENAME_EXCHANGE + Atomically exchange oldpath and newpath. Both pathnames + must exist but may be of different types (e.g., one could + be a non-empty directory and the other a symbolic link). + + RENAME_NOREPLACE + Don't overwrite newpath of the rename. Return an error if + newpath already exists. + + RENAME_NOREPLACE can't be employed together with + RENAME_EXCHANGE. + + RENAME_NOREPLACE requires support from the underlying + filesystem. Support for various filesystems was added as + follows: + + * ext4 (Linux 3.15); + + * btrfs, tmpfs, and cifs (Linux 3.17); + + * xfs (Linux 4.0); + + * Support for many other filesystems was added in Linux + 4.9, including ext2, minix, reiserfs, jfs, vfat, and + bpf. + + RENAME_WHITEOUT (since Linux 3.18) + This operation makes sense only for overlay/union + filesystem implementations. + + Specifying RENAME_WHITEOUT creates a "whiteout" object at + the source of the rename at the same time as performing + the rename. The whole operation is atomic, so that if the + rename succeeds then the whiteout will also have been + created. + + A "whiteout" is an object that has special meaning in + union/overlay filesystem constructs. In these constructs, + multiple layers exist and only the top one is ever + modified. A whiteout on an upper layer will effectively + hide a matching file in the lower layer, making it appear + as if the file didn't exist. + + When a file that exists on the lower layer is renamed, the + file is first copied up (if not already on the upper + layer) and then renamed on the upper, read-write layer. + At the same time, the source file needs to be "whiteouted" + (so that the version of the source file in the lower layer + is rendered invisible). The whole operation needs to be + done atomically. + + When not part of a union/overlay, the whiteout appears as + a character device with a {0,0} device number. (Note that + other union/overlay implementations may employ different + methods for storing whiteout entries; specifically, BSD + union mount employs a separate inode type, DT_WHT, which, + while supported by some filesystems available in Linux, + such as CODA and XFS, is ignored by the kernel's whiteout + support code, as of Linux 4.19, at least.) + + RENAME_WHITEOUT requires the same privileges as creating a + device node (i.e., the CAP_MKNOD capability). + + RENAME_WHITEOUT can't be employed together with + RENAME_EXCHANGE. + + RENAME_WHITEOUT requires support from the underlying + filesystem. Among the filesystems that support it are + tmpfs (since Linux 3.18), ext4 (since Linux 3.18), XFS + (since Linux 4.1), f2fs (since Linux 4.2), btrfs (since + Linux 4.7), and ubifs (since Linux 4.9). +*/ +const ( + RenameEmptyFlag = 0 + RenameNoReplace = 1 + RenameExchange = fs.RENAME_EXCHANGE + RenameWhiteout = 3 +) + +func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) { + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + if s := checkName(newName); s != fuse.OK { + return s + } + + switch in.Flags { + case RenameEmptyFlag: + case RenameNoReplace: + case RenameExchange: + case RenameWhiteout: + return fuse.ENOTSUP + default: + return fuse.EINVAL + } + + oldDir, code := wfs.inodeToPath.GetPath(in.NodeId) + if code != fuse.OK { + return + } + oldPath := oldDir.Child(oldName) + newDir, code := wfs.inodeToPath.GetPath(in.Newdir) + if code != fuse.OK { + return + } + newPath := newDir.Child(newName) + + glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath) + + // update remote filer + err := wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + request := &filer_pb.StreamRenameEntryRequest{ + OldDirectory: string(oldDir), + OldName: oldName, + NewDirectory: string(newDir), + NewName: newName, + Signatures: []int32{wfs.signature}, + } + + stream, err := client.StreamRenameEntry(ctx, request) + if err != nil { + code = fuse.EIO + return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err) + } + + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + if strings.Contains(recvErr.Error(), "not empty") { + code = fuse.Status(syscall.ENOTEMPTY) + } else if strings.Contains(recvErr.Error(), "not directory") { + code = fuse.ENOTDIR + } + return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr) + } + } + + if err = wfs.handleRenameResponse(ctx, resp); err != nil { + glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err) + return err + } + + } + + return nil + + }) + if err != nil { + glog.V(0).Infof("Link: %v", err) + return + } + + return fuse.OK + +} + +func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error { + // comes from filer StreamRenameEntry, can only be create or delete entry + + glog.V(4).Infof("dir Rename %+v", resp.EventNotification) + + if resp.EventNotification.NewEntry != nil { + // with new entry, the old entry name also exists. This is the first step to create new entry + newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry) + if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry, false); err != nil { + return err + } + + oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath) + oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name + + oldPath := oldParent.Child(oldName) + newPath := newParent.Child(newName) + + replacedInode := wfs.inodeToPath.MovePath(oldPath, newPath) + // invalidate attr and data + if replacedInode > 0 { + wfs.fuseServer.InodeNotify(replacedInode, 0, -1) + } + + } else if resp.EventNotification.OldEntry != nil { + // without new entry, only old entry name exists. This is the second step to delete old entry + if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil, resp.EventNotification.DeleteChunks); err != nil { + return err + } + } + + return nil + +} diff --git a/weed/mount/weedfs_stats.go b/weed/mount/weedfs_stats.go new file mode 100644 index 000000000..21f664889 --- /dev/null +++ b/weed/mount/weedfs_stats.go @@ -0,0 +1,87 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "math" + "time" +) + +const blockSize = 512 + +type statsCache struct { + filer_pb.StatisticsResponse + lastChecked int64 // unix time in seconds +} + +func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.StatfsOut) (code fuse.Status) { + + // glog.V(4).Infof("reading fs stats") + + if wfs.stats.lastChecked < time.Now().Unix()-20 { + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.StatisticsRequest{ + Collection: wfs.option.Collection, + Replication: wfs.option.Replication, + Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), + DiskType: string(wfs.option.DiskType), + } + + glog.V(4).Infof("reading filer stats: %+v", request) + resp, err := client.Statistics(context.Background(), request) + if err != nil { + glog.V(0).Infof("reading filer stats %v: %v", request, err) + return err + } + glog.V(4).Infof("read filer stats: %+v", resp) + + wfs.stats.TotalSize = resp.TotalSize + wfs.stats.UsedSize = resp.UsedSize + wfs.stats.FileCount = resp.FileCount + wfs.stats.lastChecked = time.Now().Unix() + + return nil + }) + if err != nil { + glog.V(0).Infof("filer Statistics: %v", err) + return fuse.OK + } + } + + totalDiskSize := wfs.stats.TotalSize + usedDiskSize := wfs.stats.UsedSize + actualFileCount := wfs.stats.FileCount + + if wfs.option.Quota > 0 && totalDiskSize > uint64(wfs.option.Quota) { + totalDiskSize = uint64(wfs.option.Quota) + if usedDiskSize > totalDiskSize { + totalDiskSize = usedDiskSize + } + } + + // Compute the total number of available blocks + out.Blocks = totalDiskSize / blockSize + + // Compute the number of used blocks + numBlocks := uint64(usedDiskSize / blockSize) + + // Report the number of free and available blocks for the block size + out.Bfree = out.Blocks - numBlocks + out.Bavail = out.Blocks - numBlocks + out.Bsize = uint32(blockSize) + + // Report the total number of possible files in the file system (and those free) + out.Files = math.MaxInt64 + out.Ffree = math.MaxInt64 - actualFileCount + + // Report the maximum length of a name and the minimum fragment size + out.NameLen = 1024 + out.Frsize = uint32(blockSize) + + return fuse.OK +} diff --git a/weed/mount/weedfs_symlink.go b/weed/mount/weedfs_symlink.go new file mode 100644 index 000000000..b8be55011 --- /dev/null +++ b/weed/mount/weedfs_symlink.go @@ -0,0 +1,88 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "syscall" + "time" +) + +/** Create a symbolic link */ +func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target string, name string, out *fuse.EntryOut) (code fuse.Status) { + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + if s := checkName(name); s != fuse.OK { + return s + } + + dirPath, code := wfs.inodeToPath.GetPath(header.NodeId) + if code != fuse.OK { + return + } + entryFullPath := dirPath.Child(name) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirPath), + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(os.FileMode(0777) | os.ModeSymlink), + Uid: header.Uid, + Gid: header.Gid, + SymlinkTarget: target, + }, + }, + Signatures: []int32{wfs.signature}, + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.CreateEntry(client, request); err != nil { + return fmt.Errorf("symlink %s: %v", entryFullPath, err) + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + if err != nil { + glog.V(0).Infof("Symlink %s => %s: %v", entryFullPath, target, err) + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(entryFullPath, request.Entry.Attributes.Crtime, false, false, 0, true) + + wfs.outputPbEntry(out, inode, request.Entry) + + return fuse.OK +} + +func (wfs *WFS) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []byte, code fuse.Status) { + entryFullPath, code := wfs.inodeToPath.GetPath(header.NodeId) + if code != fuse.OK { + return + } + + entry, status := wfs.maybeLoadEntry(entryFullPath) + if status != fuse.OK { + return nil, status + } + if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 { + return nil, fuse.EINVAL + } + + return []byte(entry.Attributes.SymlinkTarget), fuse.OK +} diff --git a/weed/mount/weedfs_unsupported.go b/weed/mount/weedfs_unsupported.go new file mode 100644 index 000000000..2536811b8 --- /dev/null +++ b/weed/mount/weedfs_unsupported.go @@ -0,0 +1,65 @@ +package mount + +import "github.com/hanwen/go-fuse/v2/fuse" + +// https://github.com/libfuse/libfuse/blob/48ae2e72b39b6a31cb2194f6f11786b7ca06aac6/include/fuse.h#L778 + +/** + * Copy a range of data from one file to anotherNiels de Vos, 4 years ago: â€ĸ libfuse: add copy_file_range() support + * + * Performs an optimized copy between two file descriptors without the + * additional cost of transferring data through the FUSE kernel module + * to user space (glibc) and then back into the FUSE filesystem again. + * + * In case this method is not implemented, applications are expected to + * fall back to a regular file copy. (Some glibc versions did this + * emulation automatically, but the emulation has been removed from all + * glibc release branches.) + */ +func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn) (written uint32, code fuse.Status) { + return 0, fuse.ENOSYS +} + +/** + * Allocates space for an open file + * + * This function ensures that required space is allocated for specified + * file. If this function returns success then any subsequent write + * request to specified range is guaranteed not to fail because of lack + * of space on the file system media. + */ +func (wfs *WFS) Fallocate(cancel <-chan struct{}, in *fuse.FallocateIn) (code fuse.Status) { + return fuse.ENOSYS +} + +/** + * Find next data or hole after the specified offset + */ +func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekOut) fuse.Status { + return fuse.ENOSYS +} + +func (wfs *WFS) GetLk(cancel <-chan struct{}, in *fuse.LkIn, out *fuse.LkOut) (code fuse.Status) { + return fuse.ENOSYS +} + +func (wfs *WFS) SetLk(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) { + return fuse.ENOSYS +} + +func (wfs *WFS) SetLkw(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) { + return fuse.ENOSYS +} + +/** + * Check file access permissions + * + * This will be called for the access() system call. If the + * 'default_permissions' mount option is given, this method is not + * called. + * + * This method is not called under Linux kernel versions 2.4.x + */ +func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) { + return fuse.ENOSYS +} diff --git a/weed/filesys/wfs_write.go b/weed/mount/weedfs_write.go similarity index 79% rename from weed/filesys/wfs_write.go rename to weed/mount/weedfs_write.go index 42c13cfd0..723ce9c34 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/mount/weedfs_write.go @@ -1,4 +1,4 @@ -package filesys +package mount import ( "context" @@ -13,13 +13,13 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.SaveDataAsChunkFunctionType { +func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) { var fileId, host string var auth security.EncodedJwt - if err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return util.Retry("assignVolume", func() error { request := &filer_pb.AssignVolumeRequest{ Count: 1, @@ -41,10 +41,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa } fileId, auth = resp.FileId, security.EncodedJwt(resp.Auth) - loc := &filer_pb.Location{ - Url: resp.Url, - PublicUrl: resp.PublicUrl, - } + loc := resp.Location host = wfs.AdjustedUrl(loc) collection, replication = resp.Collection, resp.Replication @@ -58,7 +55,16 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa if wfs.option.VolumeServerAccess == "filerProxy" { fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId) } - uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: fileUrl, + Filename: filename, + Cipher: wfs.option.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + } + uploadResult, err, data := operation.Upload(reader, uploadOption) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) return nil, "", "", fmt.Errorf("upload data: %v", err) @@ -68,7 +74,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa return nil, "", "", fmt.Errorf("upload result: %v", uploadResult.Error) } - if !writeOnly { + if offset == 0 { wfs.chunkCache.SetChunk(fileId, data) } diff --git a/weed/mount/weedfs_xattr.go b/weed/mount/weedfs_xattr.go new file mode 100644 index 000000000..64cc0f6f0 --- /dev/null +++ b/weed/mount/weedfs_xattr.go @@ -0,0 +1,201 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" + sys "golang.org/x/sys/unix" + "runtime" + "strings" + "syscall" +) + +const ( + // https://man7.org/linux/man-pages/man7/xattr.7.html#:~:text=The%20VFS%20imposes%20limitations%20that,in%20listxattr(2)). + MAX_XATTR_NAME_SIZE = 255 + MAX_XATTR_VALUE_SIZE = 65536 + XATTR_PREFIX = "xattr-" // same as filer +) + +// GetXAttr reads an extended attribute, and should return the +// number of bytes. If the buffer is too small, return ERANGE, +// with the required buffer size. +func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (size uint32, code fuse.Status) { + + if wfs.option.DisableXAttr { + return 0, fuse.Status(syscall.ENOTSUP) + } + + //validate attr name + if len(attr) > MAX_XATTR_NAME_SIZE { + if runtime.GOOS == "darwin" { + return 0, fuse.EPERM + } else { + return 0, fuse.ERANGE + } + } + if len(attr) == 0 { + return 0, fuse.EINVAL + } + + _, _, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return 0, status + } + if entry == nil { + return 0, fuse.ENOENT + } + if entry.Extended == nil { + return 0, fuse.ENOATTR + } + data, found := entry.Extended[XATTR_PREFIX+attr] + if !found { + return 0, fuse.ENOATTR + } + if len(dest) < len(data) { + return uint32(len(data)), fuse.ERANGE + } + copy(dest, data) + + return uint32(len(data)), fuse.OK +} + +// SetXAttr writes an extended attribute. +// https://man7.org/linux/man-pages/man2/setxattr.2.html +// By default (i.e., flags is zero), the extended attribute will be +// created if it does not exist, or the value will be replaced if +// the attribute already exists. To modify these semantics, one of +// the following values can be specified in flags: +// +// XATTR_CREATE +// Perform a pure create, which fails if the named attribute +// exists already. +// +// XATTR_REPLACE +// Perform a pure replace operation, which fails if the named +// attribute does not already exist. +func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr string, data []byte) fuse.Status { + + if wfs.option.DisableXAttr { + return fuse.Status(syscall.ENOTSUP) + } + + if wfs.IsOverQuota { + return fuse.Status(syscall.ENOSPC) + } + + //validate attr name + if len(attr) > MAX_XATTR_NAME_SIZE { + if runtime.GOOS == "darwin" { + return fuse.EPERM + } else { + return fuse.ERANGE + } + } + if len(attr) == 0 { + return fuse.EINVAL + } + //validate attr value + if len(data) > MAX_XATTR_VALUE_SIZE { + if runtime.GOOS == "darwin" { + return fuse.Status(syscall.E2BIG) + } else { + return fuse.ERANGE + } + } + + path, fh, entry, status := wfs.maybeReadEntry(input.NodeId) + if status != fuse.OK { + return status + } + if fh != nil { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + } + + if entry.Extended == nil { + entry.Extended = make(map[string][]byte) + } + oldData, _ := entry.Extended[XATTR_PREFIX+attr] + switch input.Flags { + case sys.XATTR_CREATE: + if len(oldData) > 0 { + break + } + fallthrough + case sys.XATTR_REPLACE: + fallthrough + default: + entry.Extended[XATTR_PREFIX+attr] = data + } + + return wfs.saveEntry(path, entry) + +} + +// ListXAttr lists extended attributes as '\0' delimited byte +// slice, and return the number of bytes. If the buffer is too +// small, return ERANGE, with the required buffer size. +func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (n uint32, code fuse.Status) { + + if wfs.option.DisableXAttr { + return 0, fuse.Status(syscall.ENOTSUP) + } + + _, _, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return 0, status + } + if entry == nil { + return 0, fuse.ENOENT + } + if entry.Extended == nil { + return 0, fuse.OK + } + + var data []byte + for k := range entry.Extended { + if strings.HasPrefix(k, XATTR_PREFIX) { + data = append(data, k[len(XATTR_PREFIX):]...) + data = append(data, 0) + } + } + if len(dest) < len(data) { + return uint32(len(data)), fuse.ERANGE + } + + copy(dest, data) + + return uint32(len(data)), fuse.OK +} + +// RemoveXAttr removes an extended attribute. +func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) fuse.Status { + + if wfs.option.DisableXAttr { + return fuse.Status(syscall.ENOTSUP) + } + + if len(attr) == 0 { + return fuse.EINVAL + } + path, fh, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return status + } + if fh != nil { + fh.entryLock.Lock() + defer fh.entryLock.Unlock() + } + + if entry.Extended == nil { + return fuse.ENOATTR + } + _, found := entry.Extended[XATTR_PREFIX+attr] + + if !found { + return fuse.ENOATTR + } + + delete(entry.Extended, XATTR_PREFIX+attr) + + return wfs.saveEntry(path, entry) +} diff --git a/weed/filesys/wfs_filer_client.go b/weed/mount/wfs_filer_client.go similarity index 73% rename from weed/filesys/wfs_filer_client.go rename to weed/mount/wfs_filer_client.go index 95ebdb9b8..e8feb8342 100644 --- a/weed/filesys/wfs_filer_client.go +++ b/weed/mount/wfs_filer_client.go @@ -1,4 +1,4 @@ -package filesys +package mount import ( "github.com/chrislusf/seaweedfs/weed/glog" @@ -11,16 +11,16 @@ import ( var _ = filer_pb.FilerClient(&WFS{}) -func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err error) { +func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) { return util.Retry("filer grpc", func() error { i := wfs.option.filerIndex - n := len(wfs.option.FilerGrpcAddresses) + n := len(wfs.option.FilerAddresses) for x := 0; x < n; x++ { - filerGrpcAddress := wfs.option.FilerGrpcAddresses[i] - err = pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress() + err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) }, filerGrpcAddress, wfs.option.GrpcDialOption) diff --git a/weed/mount/wfs_save.go b/weed/mount/wfs_save.go new file mode 100644 index 000000000..0cac30453 --- /dev/null +++ b/weed/mount/wfs_save.go @@ -0,0 +1,67 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fuse" + "syscall" +) + +func (wfs *WFS) saveEntry(path util.FullPath, entry *filer_pb.Entry) (code fuse.Status) { + + parentDir, _ := path.DirAndName() + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(entry) + defer wfs.mapPbIdFromFilerToLocal(entry) + + request := &filer_pb.UpdateEntryRequest{ + Directory: parentDir, + Entry: entry, + Signatures: []int32{wfs.signature}, + } + + glog.V(1).Infof("save entry: %v", request) + _, err := client.UpdateEntry(context.Background(), request) + if err != nil { + return fmt.Errorf("UpdateEntry dir %s: %v", path, err) + } + + if err := wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("UpdateEntry dir %s: %v", path, err) + } + + return nil + }) + if err != nil { + glog.Errorf("saveEntry %s: %v", path, err) + return fuse.EIO + } + + return fuse.OK +} + +func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) { + if entry.Attributes == nil { + return + } + entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) +} +func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { + if entry.Attributes == nil { + return + } + entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) +} + +func checkName(name string) fuse.Status { + if len(name) >= 256 { + return fuse.Status(syscall.ENAMETOOLONG) + } + return fuse.OK +} diff --git a/weed/notification/gocdk_pub_sub/doc.go b/weed/notification/gocdk_pub_sub/doc.go new file mode 100644 index 000000000..d7fbb9f78 --- /dev/null +++ b/weed/notification/gocdk_pub_sub/doc.go @@ -0,0 +1,9 @@ +/* + +Package gocdk_pub_sub is for Azure Service Bus and RabbitMQ. + +The referenced "gocloud.dev/pubsub" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package gocdk_pub_sub diff --git a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go index 01c4d901f..f31b6997e 100644 --- a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go +++ b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + // Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API, // which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus, // Google Cloud PubSub, and RabbitMQ. diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index f441dcb50..b716300e2 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -3,6 +3,7 @@ package operation import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "google.golang.org/grpc" @@ -26,9 +27,11 @@ type AssignResult struct { Fid string `json:"fid,omitempty"` Url string `json:"url,omitempty"` PublicUrl string `json:"publicUrl,omitempty"` + GrpcPort int `json:"grpcPort,omitempty"` Count uint64 `json:"count,omitempty"` Error string `json:"error,omitempty"` Auth security.EncodedJwt `json:"auth,omitempty"` + Replicas []Location `json:"replicas,omitempty"` } func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) { @@ -45,7 +48,7 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest continue } - lastError = WithMasterServerClient(masterFn(), grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + lastError = WithMasterServerClient(false, masterFn(), grpcDialOption, func(masterClient master_pb.SeaweedClient) error { req := &master_pb.AssignRequest{ Count: request.Count, @@ -63,15 +66,22 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest return grpcErr } + if resp.Error != "" { + return fmt.Errorf("assignRequest: %v", resp.Error) + } + ret.Count = resp.Count ret.Fid = resp.Fid - ret.Url = resp.Url - ret.PublicUrl = resp.PublicUrl + ret.Url = resp.Location.Url + ret.PublicUrl = resp.Location.PublicUrl + ret.GrpcPort = int(resp.Location.GrpcPort) ret.Error = resp.Error ret.Auth = security.EncodedJwt(resp.Auth) - - if resp.Error != "" { - return fmt.Errorf("assignRequest: %v", resp.Error) + for _, r := range resp.Replicas { + ret.Replicas = append(ret.Replicas, Location{ + Url: r.Url, + PublicUrl: r.PublicUrl, + }) } return nil @@ -93,9 +103,9 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest return ret, lastError } -func LookupJwt(master string, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) { +func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) { - WithMasterServerClient(master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + WithMasterServerClient(false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error { resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{ VolumeOrFileIds: []string{fileId}, @@ -123,6 +133,7 @@ type StorageOption struct { Collection string DataCenter string Rack string + DataNode string TtlSeconds int32 Fsync bool VolumeGrowthCount uint32 @@ -141,9 +152,10 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a DiskType: so.DiskType, DataCenter: so.DataCenter, Rack: so.Rack, + DataNode: so.DataNode, WritableVolumeCount: so.VolumeGrowthCount, } - if so.DataCenter != "" || so.Rack != "" { + if so.DataCenter != "" || so.Rack != "" || so.DataNode != "" { altRequest = &VolumeAssignRequest{ Count: uint64(count), Replication: so.Replication, @@ -152,6 +164,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a DiskType: so.DiskType, DataCenter: "", Rack: "", + DataNode: "", WritableVolumeCount: so.VolumeGrowthCount, } } diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index 94939f1f3..45068bbcc 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "sort" "sync" @@ -13,6 +12,7 @@ import ( "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -42,7 +42,7 @@ type ChunkManifest struct { type ChunkedFileReader struct { totalSize int64 chunkList []*ChunkInfo - master string + master pb.ServerAddress pos int64 pr *io.PipeReader pw *io.PipeWriter @@ -107,7 +107,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri return written, err } defer func() { - io.Copy(ioutil.Discard, resp.Body) + io.Copy(io.Discard, resp.Body) resp.Body.Close() }() @@ -127,7 +127,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri return io.Copy(w, resp.Body) } -func NewChunkedFileReader(chunkList []*ChunkInfo, master string, grpcDialOption grpc.DialOption) *ChunkedFileReader { +func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader { var totalSize int64 for _, chunk := range chunkList { totalSize += chunk.Size @@ -176,7 +176,7 @@ func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) { for ; chunkIndex < len(cf.chunkList); chunkIndex++ { ci := cf.chunkList[chunkIndex] // if we need read date from local volume server first? - fileUrl, jwt, lookupError := LookupFileId(func() string { + fileUrl, jwt, lookupError := LookupFileId(func() pb.ServerAddress { return cf.master }, cf.grpcDialOption, ci.Fid) if lookupError != nil { diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index 15d07a52e..587cf1d01 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "google.golang.org/grpc" "net/http" "strings" @@ -74,22 +75,23 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str return ret, err } - server_to_fileIds := make(map[string][]string) + server_to_fileIds := make(map[pb.ServerAddress][]string) for vid, result := range lookupResults { if result.Error != "" { ret = append(ret, &volume_server_pb.DeleteResult{ FileId: vid, Status: http.StatusBadRequest, - Error: err.Error()}, + Error: result.Error}, ) continue } for _, location := range result.Locations { - if _, ok := server_to_fileIds[location.Url]; !ok { - server_to_fileIds[location.Url] = make([]string, 0) + serverAddress := location.ServerAddress() + if _, ok := server_to_fileIds[serverAddress]; !ok { + server_to_fileIds[serverAddress] = make([]string, 0) } - server_to_fileIds[location.Url] = append( - server_to_fileIds[location.Url], vid_to_fileIds[vid]...) + server_to_fileIds[serverAddress] = append( + server_to_fileIds[serverAddress], vid_to_fileIds[vid]...) } } @@ -97,7 +99,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str var wg sync.WaitGroup for server, fidList := range server_to_fileIds { wg.Add(1) - go func(server string, fidList []string) { + go func(server pb.ServerAddress, fidList []string) { defer wg.Done() if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil { @@ -119,9 +121,9 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str } // DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc -func DeleteFilesAtOneVolumeServer(volumeServer string, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) (ret []*volume_server_pb.DeleteResult, err error) { +func DeleteFilesAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) (ret []*volume_server_pb.DeleteResult, err error) { - err = WithVolumeServerClient(volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err = WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { req := &volume_server_pb.BatchDeleteRequest{ FileIds: fileIds, diff --git a/weed/operation/grpc_client.go b/weed/operation/grpc_client.go index 025a65b38..9b68d2286 100644 --- a/weed/operation/grpc_client.go +++ b/weed/operation/grpc_client.go @@ -1,67 +1,27 @@ package operation import ( - "fmt" - "strconv" - "strings" - "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" ) -func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error { - - grpcAddress, err := toVolumeServerGrpcAddress(volumeServer) - if err != nil { - return fmt.Errorf("failed to parse volume server %v: %v", volumeServer, err) - } +func WithVolumeServerClient(streamingMode bool, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := volume_server_pb.NewVolumeServerClient(grpcConnection) return fn(client) - }, grpcAddress, grpcDialOption) - -} + }, volumeServer.ToGrpcAddress(), grpcDialOption) -func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err error) { - sepIndex := strings.LastIndex(volumeServer, ":") - port, err := strconv.Atoi(volumeServer[sepIndex+1:]) - if err != nil { - glog.Errorf("failed to parse volume server address: %v", volumeServer) - return "", err - } - return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil } -func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error { - - masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(masterServer) - if parseErr != nil { - return fmt.Errorf("failed to parse master %v: %v", masterServer, parseErr) - } +func WithMasterServerClient(streamingMode bool, masterServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := master_pb.NewSeaweedClient(grpcConnection) return fn(client) - }, masterGrpcAddress, grpcDialOption) - -} - -func WithFilerServerClient(filerServer string, grpcDialOption grpc.DialOption, fn func(masterClient filer_pb.SeaweedFilerClient) error) error { - - filerGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(filerServer) - if parseErr != nil { - return fmt.Errorf("failed to parse filer %v: %v", filerGrpcAddress, parseErr) - } - - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { - client := filer_pb.NewSeaweedFilerClient(grpcConnection) - return fn(client) - }, filerGrpcAddress, grpcDialOption) + }, masterServer.ToGrpcAddress(), grpcDialOption) } diff --git a/weed/operation/lookup.go b/weed/operation/lookup.go index 8717f6b36..1eb5dd320 100644 --- a/weed/operation/lookup.go +++ b/weed/operation/lookup.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "google.golang.org/grpc" "math/rand" "strings" @@ -15,7 +16,13 @@ import ( type Location struct { Url string `json:"url,omitempty"` PublicUrl string `json:"publicUrl,omitempty"` + GrpcPort int `json:"grpcPort,omitempty"` } + +func (l *Location) ServerAddress() pb.ServerAddress { + return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort) +} + type LookupResult struct { VolumeOrFileId string `json:"volumeOrFileId,omitempty"` Locations []Location `json:"locations,omitempty"` @@ -72,7 +79,7 @@ func LookupVolumeIds(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids //only query unknown_vids - err := WithMasterServerClient(masterFn(), grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + err := WithMasterServerClient(false, masterFn(), grpcDialOption, func(masterClient master_pb.SeaweedClient) error { req := &master_pb.LookupVolumeRequest{ VolumeOrFileIds: unknown_vids, @@ -89,6 +96,7 @@ func LookupVolumeIds(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids locations = append(locations, Location{ Url: loc.Url, PublicUrl: loc.PublicUrl, + GrpcPort: int(loc.GrpcPort), }) } if vidLocations.Error != "" { diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index d7e8a4162..2b44b3b26 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -53,7 +53,16 @@ func TestCreateNeedleFromRequest(t *testing.T) { assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip") fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize) } - uploadResult, err, data := Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader([]byte(textContent)), false, "", nil, "") + uploadOption := &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0", + Filename: "t.txt", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: "", + } + uploadResult, err, data := Upload(bytes.NewReader([]byte(textContent)), uploadOption) if len(data) != len(textContent) { t.Errorf("data actual %d expected %d", len(data), len(textContent)) } @@ -72,7 +81,16 @@ func TestCreateNeedleFromRequest(t *testing.T) { fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize) } gzippedData, _ := util.GzipData([]byte(textContent)) - Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "") + uploadOption := &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0", + Filename: "t.txt", + Cipher: false, + IsInputCompressed: true, + MimeType: "text/plain", + PairMap: nil, + Jwt: "", + } + Upload(bytes.NewReader(gzippedData), uploadOption) } /* diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 87c5e4279..648df174a 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -1,6 +1,7 @@ package operation import ( + "github.com/chrislusf/seaweedfs/weed/pb" "io" "mime" "net/url" @@ -39,7 +40,7 @@ type SubmitResult struct { Error string `json:"error,omitempty"` } -type GetMasterFn func() string +type GetMasterFn func() pb.ServerAddress func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []FilePart, replication string, collection string, dataCenter string, ttl string, diskType string, maxMB int, usePublicUrl bool) ([]SubmitResult, error) { results := make([]SubmitResult, len(files)) @@ -206,7 +207,16 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption) } } else { - ret, e, _ := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt) + uploadOption := &UploadOption{ + UploadUrl: fileUrl, + Filename: baseName, + Cipher: false, + IsInputCompressed: false, + MimeType: fi.MimeType, + PairMap: nil, + Jwt: jwt, + } + ret, e, _ := Upload(fi.Reader, uploadOption) if e != nil { return 0, e } @@ -219,7 +229,16 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn, fileUrl string, jwt security.EncodedJwt, ) (size uint32, e error) { glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") - uploadResult, uploadError, _ := Upload(fileUrl, filename, false, reader, false, "", nil, jwt) + uploadOption := &UploadOption{ + UploadUrl: fileUrl, + Filename: filename, + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: jwt, + } + uploadResult, uploadError, _ := Upload(reader, uploadOption) if uploadError != nil { return 0, uploadError } @@ -236,6 +255,15 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s q := u.Query() q.Set("cm", "true") u.RawQuery = q.Encode() - _, e = UploadData(u.String(), manifest.Name, false, buf, false, "application/json", nil, jwt) + uploadOption := &UploadOption{ + UploadUrl: u.String(), + Filename: manifest.Name, + Cipher: false, + IsInputCompressed: false, + MimeType: "application/json", + PairMap: nil, + Jwt: jwt, + } + _, e = UploadData(buf, uploadOption) return e } diff --git a/weed/operation/sync_volume.go b/weed/operation/sync_volume.go index 5562f12ab..de71a198d 100644 --- a/weed/operation/sync_volume.go +++ b/weed/operation/sync_volume.go @@ -2,13 +2,14 @@ package operation import ( "context" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "google.golang.org/grpc" ) -func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) { +func GetVolumeSyncStatus(server pb.ServerAddress, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) { - WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + WithVolumeServerClient(false, server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{ VolumeId: vid, diff --git a/weed/operation/tail_volume.go b/weed/operation/tail_volume.go index e3f2c0664..d3449873b 100644 --- a/weed/operation/tail_volume.go +++ b/weed/operation/tail_volume.go @@ -3,6 +3,7 @@ package operation import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "google.golang.org/grpc" @@ -21,13 +22,13 @@ func TailVolume(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid needle return fmt.Errorf("unable to locate volume %d", vid) } - volumeServer := lookup.Locations[0].Url + volumeServer := lookup.Locations[0].ServerAddress() return TailVolumeFromSource(volumeServer, grpcDialOption, vid, sinceNs, timeoutSeconds, fn) } -func TailVolumeFromSource(volumeServer string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error { - return WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { +func TailVolumeFromSource(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error { + return WithVolumeServerClient(true, volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 8e7c6f733..3d41d2eb5 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -4,22 +4,31 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" "io" - "io/ioutil" "mime" "mime/multipart" + "net" "net/http" "net/textproto" "path/filepath" "strings" "time" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/util" ) +type UploadOption struct { + UploadUrl string + Filename string + Cipher bool + IsInputCompressed bool + MimeType string + PairMap map[string]string + Jwt security.EncodedJwt +} + type UploadResult struct { Name string `json:"name,omitempty"` Size uint32 `json:"size,omitempty"` @@ -57,68 +66,72 @@ var ( func init() { HttpClient = &http.Client{Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 10 * time.Second, + }).DialContext, MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, }} } -var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) +var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "") // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { - uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) +func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { + uploadResult, err = retriedUploadData(data, option) return } // Upload sends a POST request to a volume server to upload the content with fast compression -func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) { - uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt) +func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { + uploadResult, err, data = doUpload(reader, option) return } -func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) { +func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { bytesReader, ok := reader.(*util.BytesReader) if ok { data = bytesReader.Bytes } else { - data, err = ioutil.ReadAll(reader) + data, err = io.ReadAll(reader) if err != nil { err = fmt.Errorf("read input: %v", err) return } } - uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) + uploadResult, uploadErr := retriedUploadData(data, option) return uploadResult, uploadErr, data } -func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { +func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { for i := 0; i < 3; i++ { - uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) + uploadResult, err = doUploadData(data, option) if err == nil { uploadResult.RetryCount = i return } else { - glog.Warningf("uploading to %s: %v", uploadUrl, err) + glog.Warningf("uploading to %s: %v", option.UploadUrl, err) } time.Sleep(time.Millisecond * time.Duration(237*(i+1))) } return } -func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { - contentIsGzipped := isInputCompressed +func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { + contentIsGzipped := option.IsInputCompressed shouldGzipNow := false - if !isInputCompressed { - if mtype == "" { - mtype = http.DetectContentType(data) - // println("detect1 mimetype to", mtype) - if mtype == "application/octet-stream" { - mtype = "" + if !option.IsInputCompressed { + if option.MimeType == "" { + option.MimeType = http.DetectContentType(data) + // println("detect1 mimetype to", MimeType) + if option.MimeType == "application/octet-stream" { + option.MimeType = "" } } - if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed { + if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { shouldGzipNow = true - } else if !iAmSure && mtype == "" && len(data) > 16*1024 { + } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 { var compressed []byte compressed, err = util.GzipData(data[0:128]) shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% @@ -131,14 +144,14 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i // this could be double copying clearDataLen = len(data) clearData := data - if shouldGzipNow && !cipher { + if shouldGzipNow && !option.Cipher { compressed, compressErr := util.GzipData(data) // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) if compressErr == nil { data = compressed contentIsGzipped = true } - } else if isInputCompressed { + } else if option.IsInputCompressed { // just to get the clear data length clearData, err = util.DecompressData(data) if err == nil { @@ -146,7 +159,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i } } - if cipher { + if option.Cipher { // encrypt(gzip(data)) // encrypt @@ -158,23 +171,39 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i } // upload data - uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + uploadResult, err = upload_content(func(w io.Writer) (err error) { _, err = w.Write(encryptedData) return - }, "", false, len(encryptedData), "", nil, jwt) + }, len(encryptedData), &UploadOption{ + UploadUrl: option.UploadUrl, + Filename: "", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: option.Jwt, + }) if uploadResult == nil { return } - uploadResult.Name = filename - uploadResult.Mime = mtype + uploadResult.Name = option.Filename + uploadResult.Mime = option.MimeType uploadResult.CipherKey = cipherKey uploadResult.Size = uint32(clearDataLen) } else { // upload data - uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + uploadResult, err = upload_content(func(w io.Writer) (err error) { _, err = w.Write(data) return - }, filename, contentIsGzipped, len(data), mtype, pairMap, jwt) + }, len(data), &UploadOption{ + UploadUrl: option.UploadUrl, + Filename: option.Filename, + Cipher: false, + IsInputCompressed: contentIsGzipped, + MimeType: option.MimeType, + PairMap: option.PairMap, + Jwt: option.Jwt, + }) if uploadResult == nil { return } @@ -187,20 +216,21 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i return uploadResult, err } -func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { +func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) { buf := GetBuffer() defer PutBuffer(buf) body_writer := multipart.NewWriter(buf) h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename))) - h.Set("Idempotency-Key", uploadUrl) - if mtype == "" { - mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))) + filename := fileNameEscaper.Replace(option.Filename) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename)) + h.Set("Idempotency-Key", option.UploadUrl) + if option.MimeType == "" { + option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename))) } - if mtype != "" { - h.Set("Content-Type", mtype) + if option.MimeType != "" { + h.Set("Content-Type", option.MimeType) } - if isGzipped { + if option.IsInputCompressed { h.Set("Content-Encoding", "gzip") } @@ -219,28 +249,29 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error return nil, err } - req, postErr := http.NewRequest("POST", uploadUrl, bytes.NewReader(buf.Bytes())) + req, postErr := http.NewRequest("POST", option.UploadUrl, bytes.NewReader(buf.Bytes())) if postErr != nil { - glog.V(1).Infof("create upload request %s: %v", uploadUrl, postErr) - return nil, fmt.Errorf("create upload request %s: %v", uploadUrl, postErr) + glog.V(1).Infof("create upload request %s: %v", option.UploadUrl, postErr) + return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr) } req.Header.Set("Content-Type", content_type) - for k, v := range pairMap { + for k, v := range option.PairMap { req.Header.Set(k, v) } - if jwt != "" { - req.Header.Set("Authorization", "BEARER "+string(jwt)) + if option.Jwt != "" { + req.Header.Set("Authorization", "BEARER "+string(option.Jwt)) } // print("+") resp, post_err := HttpClient.Do(req) if post_err != nil { if strings.Contains(post_err.Error(), "connection reset by peer") || strings.Contains(post_err.Error(), "use of closed network connection") { + glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr) resp, post_err = HttpClient.Do(req) } } if post_err != nil { - return nil, fmt.Errorf("upload %s %d bytes to %v: %v", filename, originalDataSize, uploadUrl, post_err) + return nil, fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err) } // print("-") defer util.CloseResponse(resp) @@ -252,18 +283,18 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error return &ret, nil } - resp_body, ra_err := ioutil.ReadAll(resp.Body) + resp_body, ra_err := io.ReadAll(resp.Body) if ra_err != nil { - return nil, fmt.Errorf("read response body %v: %v", uploadUrl, ra_err) + return nil, fmt.Errorf("read response body %v: %v", option.UploadUrl, ra_err) } unmarshal_err := json.Unmarshal(resp_body, &ret) if unmarshal_err != nil { - glog.Errorf("unmarshal %s: %v", uploadUrl, string(resp_body)) - return nil, fmt.Errorf("unmarshal %v: %v", uploadUrl, unmarshal_err) + glog.Errorf("unmarshal %s: %v", option.UploadUrl, string(resp_body)) + return nil, fmt.Errorf("unmarshal %v: %v", option.UploadUrl, unmarshal_err) } if ret.Error != "" { - return nil, fmt.Errorf("unmarshalled error %v: %v", uploadUrl, ret.Error) + return nil, fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error) } ret.ETag = etag ret.ContentMd5 = resp.Header.Get("Content-MD5") diff --git a/weed/pb/Makefile b/weed/pb/Makefile index 72c738135..a8992bde2 100644 --- a/weed/pb/Makefile +++ b/weed/pb/Makefile @@ -3,11 +3,13 @@ all: gen .PHONY : gen gen: - protoc master.proto --go_out=plugins=grpc:./master_pb --go_opt=paths=source_relative - protoc volume_server.proto --go_out=plugins=grpc:./volume_server_pb --go_opt=paths=source_relative - protoc filer.proto --go_out=plugins=grpc:./filer_pb --go_opt=paths=source_relative - protoc remote.proto --go_out=plugins=grpc:./remote_pb --go_opt=paths=source_relative - protoc iam.proto --go_out=plugins=grpc:./iam_pb --go_opt=paths=source_relative - protoc messaging.proto --go_out=plugins=grpc:./messaging_pb --go_opt=paths=source_relative + protoc master.proto --go_out=./master_pb --go-grpc_out=./master_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc volume_server.proto --go_out=./volume_server_pb --go-grpc_out=./volume_server_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc filer.proto --go_out=./filer_pb --go-grpc_out=./filer_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc remote.proto --go_out=./remote_pb --go-grpc_out=./remote_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc iam.proto --go_out=./iam_pb --go-grpc_out=./iam_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc mount.proto --go_out=./mount_pb --go-grpc_out=./mount_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc s3.proto --go_out=./s3_pb --go-grpc_out=./s3_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative + protoc messaging.proto --go_out=./messaging_pb --go-grpc_out=./messaging_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative # protoc filer.proto --java_out=../../other/java/client/src/main/java cp filer.proto ../../other/java/client/src/main/proto diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index bb461936c..bd0932cb8 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -30,6 +30,8 @@ service SeaweedFiler { rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) { } + rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) { + } rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { } @@ -46,6 +48,9 @@ service SeaweedFiler { rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { } + rpc Ping (PingRequest) returns (PingResponse) { + } + rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) { } @@ -67,7 +72,7 @@ service SeaweedFiler { rpc KvPut (KvPutRequest) returns (KvPutResponse) { } - rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) { + rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) { } } @@ -112,6 +117,7 @@ message Entry { bytes content = 9; // if not empty, the file content RemoteEntry remote_entry = 10; + int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled. } message FullEntry { @@ -160,14 +166,13 @@ message FuseAttributes { uint32 gid = 5; int64 crtime = 6; // unix time in seconds string mime = 7; - string replication = 8; - string collection = 9; int32 ttl_sec = 10; string user_name = 11; // for hdfs repeated string group_name = 12; // for hdfs string symlink_target = 13; bytes md5 = 14; - string disk_type = 15; + uint32 rdev = 16; + uint64 inode = 17; } message CreateEntryRequest { @@ -176,6 +181,7 @@ message CreateEntryRequest { bool o_excl = 3; bool is_from_other_cluster = 4; repeated int32 signatures = 5; + bool skip_check_parent_directory = 6; } message CreateEntryResponse { @@ -225,6 +231,18 @@ message AtomicRenameEntryRequest { message AtomicRenameEntryResponse { } +message StreamRenameEntryRequest { + string old_directory = 1; + string old_name = 2; + string new_directory = 3; + string new_name = 4; + repeated int32 signatures = 5; +} +message StreamRenameEntryResponse { + string directory = 1; + EventNotification event_notification = 2; + int64 ts_ns = 3; +} message AssignVolumeRequest { int32 count = 1; string collection = 2; @@ -233,18 +251,18 @@ message AssignVolumeRequest { string data_center = 5; string path = 6; string rack = 7; + string data_node = 9; string disk_type = 8; } message AssignVolumeResponse { string file_id = 1; - string url = 2; - string public_url = 3; int32 count = 4; string auth = 5; string collection = 6; string replication = 7; string error = 8; + Location location = 9; } message LookupVolumeRequest { @@ -258,6 +276,7 @@ message Locations { message Location { string url = 1; string public_url = 2; + uint32 grpc_port = 3; } message LookupVolumeResponse { map locations_map = 1; @@ -292,6 +311,16 @@ message StatisticsResponse { uint64 file_count = 6; } +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +} + message GetFilerConfigurationRequest { } message GetFilerConfigurationResponse { @@ -306,6 +335,7 @@ message GetFilerConfigurationResponse { int32 metrics_interval_sec = 10; string version = 11; string cluster_id = 12; + string filer_group = 13; } message SubscribeMetadataRequest { @@ -313,6 +343,9 @@ message SubscribeMetadataRequest { string path_prefix = 2; int64 since_ns = 3; int32 signature = 4; + repeated string path_prefixes = 6; + int32 client_id = 7; + int64 until_ns = 8; } message SubscribeMetadataResponse { string directory = 1; @@ -381,6 +414,9 @@ message FilerConf { bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; + string data_center = 9; + string rack = 10; + string data_node = 11; } repeated PathConf locations = 2; } @@ -388,10 +424,10 @@ message FilerConf { ///////////////////////// // Remote Storage related ///////////////////////// -message DownloadToLocalRequest { +message CacheRemoteObjectToLocalClusterRequest { string directory = 1; string name = 2; } -message DownloadToLocalResponse { +message CacheRemoteObjectToLocalClusterResponse { Entry entry = 1; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 3d0197a64..4ac560855 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -1,17 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: filer.proto package filer_pb import ( - context "context" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -25,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type LookupDirectoryEntryRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -350,6 +341,7 @@ type Entry struct { HardLinkCounter int32 `protobuf:"varint,8,opt,name=hard_link_counter,json=hardLinkCounter,proto3" json:"hard_link_counter,omitempty"` // only exists in hard link meta data Content []byte `protobuf:"bytes,9,opt,name=content,proto3" json:"content,omitempty"` // if not empty, the file content RemoteEntry *RemoteEntry `protobuf:"bytes,10,opt,name=remote_entry,json=remoteEntry,proto3" json:"remote_entry,omitempty"` + Quota int64 `protobuf:"varint,11,opt,name=quota,proto3" json:"quota,omitempty"` // for bucket only. Positive/Negative means enabled/disabled. } func (x *Entry) Reset() { @@ -447,6 +439,13 @@ func (x *Entry) GetRemoteEntry() *RemoteEntry { return nil } +func (x *Entry) GetQuota() int64 { + if x != nil { + return x.Quota + } + return 0 +} + type FullEntry struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -838,14 +837,13 @@ type FuseAttributes struct { Gid uint32 `protobuf:"varint,5,opt,name=gid,proto3" json:"gid,omitempty"` Crtime int64 `protobuf:"varint,6,opt,name=crtime,proto3" json:"crtime,omitempty"` // unix time in seconds Mime string `protobuf:"bytes,7,opt,name=mime,proto3" json:"mime,omitempty"` - Replication string `protobuf:"bytes,8,opt,name=replication,proto3" json:"replication,omitempty"` - Collection string `protobuf:"bytes,9,opt,name=collection,proto3" json:"collection,omitempty"` TtlSec int32 `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec,proto3" json:"ttl_sec,omitempty"` UserName string `protobuf:"bytes,11,opt,name=user_name,json=userName,proto3" json:"user_name,omitempty"` // for hdfs GroupName []string `protobuf:"bytes,12,rep,name=group_name,json=groupName,proto3" json:"group_name,omitempty"` // for hdfs SymlinkTarget string `protobuf:"bytes,13,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"` Md5 []byte `protobuf:"bytes,14,opt,name=md5,proto3" json:"md5,omitempty"` - DiskType string `protobuf:"bytes,15,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + Rdev uint32 `protobuf:"varint,16,opt,name=rdev,proto3" json:"rdev,omitempty"` + Inode uint64 `protobuf:"varint,17,opt,name=inode,proto3" json:"inode,omitempty"` } func (x *FuseAttributes) Reset() { @@ -929,20 +927,6 @@ func (x *FuseAttributes) GetMime() string { return "" } -func (x *FuseAttributes) GetReplication() string { - if x != nil { - return x.Replication - } - return "" -} - -func (x *FuseAttributes) GetCollection() string { - if x != nil { - return x.Collection - } - return "" -} - func (x *FuseAttributes) GetTtlSec() int32 { if x != nil { return x.TtlSec @@ -978,11 +962,18 @@ func (x *FuseAttributes) GetMd5() []byte { return nil } -func (x *FuseAttributes) GetDiskType() string { +func (x *FuseAttributes) GetRdev() uint32 { if x != nil { - return x.DiskType + return x.Rdev } - return "" + return 0 +} + +func (x *FuseAttributes) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 } type CreateEntryRequest struct { @@ -990,11 +981,12 @@ type CreateEntryRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` - Entry *Entry `protobuf:"bytes,2,opt,name=entry,proto3" json:"entry,omitempty"` - OExcl bool `protobuf:"varint,3,opt,name=o_excl,json=oExcl,proto3" json:"o_excl,omitempty"` - IsFromOtherCluster bool `protobuf:"varint,4,opt,name=is_from_other_cluster,json=isFromOtherCluster,proto3" json:"is_from_other_cluster,omitempty"` - Signatures []int32 `protobuf:"varint,5,rep,packed,name=signatures,proto3" json:"signatures,omitempty"` + Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` + Entry *Entry `protobuf:"bytes,2,opt,name=entry,proto3" json:"entry,omitempty"` + OExcl bool `protobuf:"varint,3,opt,name=o_excl,json=oExcl,proto3" json:"o_excl,omitempty"` + IsFromOtherCluster bool `protobuf:"varint,4,opt,name=is_from_other_cluster,json=isFromOtherCluster,proto3" json:"is_from_other_cluster,omitempty"` + Signatures []int32 `protobuf:"varint,5,rep,packed,name=signatures,proto3" json:"signatures,omitempty"` + SkipCheckParentDirectory bool `protobuf:"varint,6,opt,name=skip_check_parent_directory,json=skipCheckParentDirectory,proto3" json:"skip_check_parent_directory,omitempty"` } func (x *CreateEntryRequest) Reset() { @@ -1064,6 +1056,13 @@ func (x *CreateEntryRequest) GetSignatures() []int32 { return nil } +func (x *CreateEntryRequest) GetSkipCheckParentDirectory() bool { + if x != nil { + return x.SkipCheckParentDirectory + } + return false +} + type CreateEntryResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1581,6 +1580,148 @@ func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return file_filer_proto_rawDescGZIP(), []int{21} } +type StreamRenameEntryRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory,proto3" json:"old_directory,omitempty"` + OldName string `protobuf:"bytes,2,opt,name=old_name,json=oldName,proto3" json:"old_name,omitempty"` + NewDirectory string `protobuf:"bytes,3,opt,name=new_directory,json=newDirectory,proto3" json:"new_directory,omitempty"` + NewName string `protobuf:"bytes,4,opt,name=new_name,json=newName,proto3" json:"new_name,omitempty"` + Signatures []int32 `protobuf:"varint,5,rep,packed,name=signatures,proto3" json:"signatures,omitempty"` +} + +func (x *StreamRenameEntryRequest) Reset() { + *x = StreamRenameEntryRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamRenameEntryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamRenameEntryRequest) ProtoMessage() {} + +func (x *StreamRenameEntryRequest) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamRenameEntryRequest.ProtoReflect.Descriptor instead. +func (*StreamRenameEntryRequest) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{22} +} + +func (x *StreamRenameEntryRequest) GetOldDirectory() string { + if x != nil { + return x.OldDirectory + } + return "" +} + +func (x *StreamRenameEntryRequest) GetOldName() string { + if x != nil { + return x.OldName + } + return "" +} + +func (x *StreamRenameEntryRequest) GetNewDirectory() string { + if x != nil { + return x.NewDirectory + } + return "" +} + +func (x *StreamRenameEntryRequest) GetNewName() string { + if x != nil { + return x.NewName + } + return "" +} + +func (x *StreamRenameEntryRequest) GetSignatures() []int32 { + if x != nil { + return x.Signatures + } + return nil +} + +type StreamRenameEntryResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` + EventNotification *EventNotification `protobuf:"bytes,2,opt,name=event_notification,json=eventNotification,proto3" json:"event_notification,omitempty"` + TsNs int64 `protobuf:"varint,3,opt,name=ts_ns,json=tsNs,proto3" json:"ts_ns,omitempty"` +} + +func (x *StreamRenameEntryResponse) Reset() { + *x = StreamRenameEntryResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamRenameEntryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamRenameEntryResponse) ProtoMessage() {} + +func (x *StreamRenameEntryResponse) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamRenameEntryResponse.ProtoReflect.Descriptor instead. +func (*StreamRenameEntryResponse) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{23} +} + +func (x *StreamRenameEntryResponse) GetDirectory() string { + if x != nil { + return x.Directory + } + return "" +} + +func (x *StreamRenameEntryResponse) GetEventNotification() *EventNotification { + if x != nil { + return x.EventNotification + } + return nil +} + +func (x *StreamRenameEntryResponse) GetTsNs() int64 { + if x != nil { + return x.TsNs + } + return 0 +} + type AssignVolumeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1593,13 +1734,14 @@ type AssignVolumeRequest struct { DataCenter string `protobuf:"bytes,5,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` Path string `protobuf:"bytes,6,opt,name=path,proto3" json:"path,omitempty"` Rack string `protobuf:"bytes,7,opt,name=rack,proto3" json:"rack,omitempty"` + DataNode string `protobuf:"bytes,9,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` DiskType string `protobuf:"bytes,8,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` } func (x *AssignVolumeRequest) Reset() { *x = AssignVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[22] + mi := &file_filer_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1612,7 +1754,7 @@ func (x *AssignVolumeRequest) String() string { func (*AssignVolumeRequest) ProtoMessage() {} func (x *AssignVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[22] + mi := &file_filer_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1625,7 +1767,7 @@ func (x *AssignVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AssignVolumeRequest.ProtoReflect.Descriptor instead. func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{22} + return file_filer_proto_rawDescGZIP(), []int{24} } func (x *AssignVolumeRequest) GetCount() int32 { @@ -1677,6 +1819,13 @@ func (x *AssignVolumeRequest) GetRack() string { return "" } +func (x *AssignVolumeRequest) GetDataNode() string { + if x != nil { + return x.DataNode + } + return "" +} + func (x *AssignVolumeRequest) GetDiskType() string { if x != nil { return x.DiskType @@ -1689,20 +1838,19 @@ type AssignVolumeResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId,proto3" json:"file_id,omitempty"` - Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` - PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` - Count int32 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` - Auth string `protobuf:"bytes,5,opt,name=auth,proto3" json:"auth,omitempty"` - Collection string `protobuf:"bytes,6,opt,name=collection,proto3" json:"collection,omitempty"` - Replication string `protobuf:"bytes,7,opt,name=replication,proto3" json:"replication,omitempty"` - Error string `protobuf:"bytes,8,opt,name=error,proto3" json:"error,omitempty"` + FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId,proto3" json:"file_id,omitempty"` + Count int32 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` + Auth string `protobuf:"bytes,5,opt,name=auth,proto3" json:"auth,omitempty"` + Collection string `protobuf:"bytes,6,opt,name=collection,proto3" json:"collection,omitempty"` + Replication string `protobuf:"bytes,7,opt,name=replication,proto3" json:"replication,omitempty"` + Error string `protobuf:"bytes,8,opt,name=error,proto3" json:"error,omitempty"` + Location *Location `protobuf:"bytes,9,opt,name=location,proto3" json:"location,omitempty"` } func (x *AssignVolumeResponse) Reset() { *x = AssignVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[23] + mi := &file_filer_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1715,7 +1863,7 @@ func (x *AssignVolumeResponse) String() string { func (*AssignVolumeResponse) ProtoMessage() {} func (x *AssignVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[23] + mi := &file_filer_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1728,7 +1876,7 @@ func (x *AssignVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AssignVolumeResponse.ProtoReflect.Descriptor instead. func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{23} + return file_filer_proto_rawDescGZIP(), []int{25} } func (x *AssignVolumeResponse) GetFileId() string { @@ -1738,20 +1886,6 @@ func (x *AssignVolumeResponse) GetFileId() string { return "" } -func (x *AssignVolumeResponse) GetUrl() string { - if x != nil { - return x.Url - } - return "" -} - -func (x *AssignVolumeResponse) GetPublicUrl() string { - if x != nil { - return x.PublicUrl - } - return "" -} - func (x *AssignVolumeResponse) GetCount() int32 { if x != nil { return x.Count @@ -1787,6 +1921,13 @@ func (x *AssignVolumeResponse) GetError() string { return "" } +func (x *AssignVolumeResponse) GetLocation() *Location { + if x != nil { + return x.Location + } + return nil +} + type LookupVolumeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1798,7 +1939,7 @@ type LookupVolumeRequest struct { func (x *LookupVolumeRequest) Reset() { *x = LookupVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[24] + mi := &file_filer_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1811,7 +1952,7 @@ func (x *LookupVolumeRequest) String() string { func (*LookupVolumeRequest) ProtoMessage() {} func (x *LookupVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[24] + mi := &file_filer_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1824,7 +1965,7 @@ func (x *LookupVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupVolumeRequest.ProtoReflect.Descriptor instead. func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{24} + return file_filer_proto_rawDescGZIP(), []int{26} } func (x *LookupVolumeRequest) GetVolumeIds() []string { @@ -1845,7 +1986,7 @@ type Locations struct { func (x *Locations) Reset() { *x = Locations{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[25] + mi := &file_filer_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1858,7 +1999,7 @@ func (x *Locations) String() string { func (*Locations) ProtoMessage() {} func (x *Locations) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[25] + mi := &file_filer_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1871,7 +2012,7 @@ func (x *Locations) ProtoReflect() protoreflect.Message { // Deprecated: Use Locations.ProtoReflect.Descriptor instead. func (*Locations) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{25} + return file_filer_proto_rawDescGZIP(), []int{27} } func (x *Locations) GetLocations() []*Location { @@ -1888,12 +2029,13 @@ type Location struct { Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` + GrpcPort uint32 `protobuf:"varint,3,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` } func (x *Location) Reset() { *x = Location{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[26] + mi := &file_filer_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1906,7 +2048,7 @@ func (x *Location) String() string { func (*Location) ProtoMessage() {} func (x *Location) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[26] + mi := &file_filer_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1919,7 +2061,7 @@ func (x *Location) ProtoReflect() protoreflect.Message { // Deprecated: Use Location.ProtoReflect.Descriptor instead. func (*Location) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{26} + return file_filer_proto_rawDescGZIP(), []int{28} } func (x *Location) GetUrl() string { @@ -1936,6 +2078,13 @@ func (x *Location) GetPublicUrl() string { return "" } +func (x *Location) GetGrpcPort() uint32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + type LookupVolumeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1947,7 +2096,7 @@ type LookupVolumeResponse struct { func (x *LookupVolumeResponse) Reset() { *x = LookupVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[27] + mi := &file_filer_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1960,7 +2109,7 @@ func (x *LookupVolumeResponse) String() string { func (*LookupVolumeResponse) ProtoMessage() {} func (x *LookupVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[27] + mi := &file_filer_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1973,7 +2122,7 @@ func (x *LookupVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupVolumeResponse.ProtoReflect.Descriptor instead. func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{27} + return file_filer_proto_rawDescGZIP(), []int{29} } func (x *LookupVolumeResponse) GetLocationsMap() map[string]*Locations { @@ -1994,7 +2143,7 @@ type Collection struct { func (x *Collection) Reset() { *x = Collection{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[28] + mi := &file_filer_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2007,7 +2156,7 @@ func (x *Collection) String() string { func (*Collection) ProtoMessage() {} func (x *Collection) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[28] + mi := &file_filer_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2020,7 +2169,7 @@ func (x *Collection) ProtoReflect() protoreflect.Message { // Deprecated: Use Collection.ProtoReflect.Descriptor instead. func (*Collection) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{28} + return file_filer_proto_rawDescGZIP(), []int{30} } func (x *Collection) GetName() string { @@ -2042,7 +2191,7 @@ type CollectionListRequest struct { func (x *CollectionListRequest) Reset() { *x = CollectionListRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[29] + mi := &file_filer_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2055,7 +2204,7 @@ func (x *CollectionListRequest) String() string { func (*CollectionListRequest) ProtoMessage() {} func (x *CollectionListRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[29] + mi := &file_filer_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2068,7 +2217,7 @@ func (x *CollectionListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionListRequest.ProtoReflect.Descriptor instead. func (*CollectionListRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{29} + return file_filer_proto_rawDescGZIP(), []int{31} } func (x *CollectionListRequest) GetIncludeNormalVolumes() bool { @@ -2096,7 +2245,7 @@ type CollectionListResponse struct { func (x *CollectionListResponse) Reset() { *x = CollectionListResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[30] + mi := &file_filer_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2109,7 +2258,7 @@ func (x *CollectionListResponse) String() string { func (*CollectionListResponse) ProtoMessage() {} func (x *CollectionListResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[30] + mi := &file_filer_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2122,7 +2271,7 @@ func (x *CollectionListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionListResponse.ProtoReflect.Descriptor instead. func (*CollectionListResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{30} + return file_filer_proto_rawDescGZIP(), []int{32} } func (x *CollectionListResponse) GetCollections() []*Collection { @@ -2143,7 +2292,7 @@ type DeleteCollectionRequest struct { func (x *DeleteCollectionRequest) Reset() { *x = DeleteCollectionRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[31] + mi := &file_filer_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2156,7 +2305,7 @@ func (x *DeleteCollectionRequest) String() string { func (*DeleteCollectionRequest) ProtoMessage() {} func (x *DeleteCollectionRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[31] + mi := &file_filer_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2169,7 +2318,7 @@ func (x *DeleteCollectionRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteCollectionRequest.ProtoReflect.Descriptor instead. func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{31} + return file_filer_proto_rawDescGZIP(), []int{33} } func (x *DeleteCollectionRequest) GetCollection() string { @@ -2188,7 +2337,7 @@ type DeleteCollectionResponse struct { func (x *DeleteCollectionResponse) Reset() { *x = DeleteCollectionResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[32] + mi := &file_filer_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2201,7 +2350,7 @@ func (x *DeleteCollectionResponse) String() string { func (*DeleteCollectionResponse) ProtoMessage() {} func (x *DeleteCollectionResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[32] + mi := &file_filer_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2214,7 +2363,7 @@ func (x *DeleteCollectionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteCollectionResponse.ProtoReflect.Descriptor instead. func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{32} + return file_filer_proto_rawDescGZIP(), []int{34} } type StatisticsRequest struct { @@ -2231,7 +2380,7 @@ type StatisticsRequest struct { func (x *StatisticsRequest) Reset() { *x = StatisticsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[33] + mi := &file_filer_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2244,7 +2393,7 @@ func (x *StatisticsRequest) String() string { func (*StatisticsRequest) ProtoMessage() {} func (x *StatisticsRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[33] + mi := &file_filer_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2257,7 +2406,7 @@ func (x *StatisticsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatisticsRequest.ProtoReflect.Descriptor instead. func (*StatisticsRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{33} + return file_filer_proto_rawDescGZIP(), []int{35} } func (x *StatisticsRequest) GetReplication() string { @@ -2301,7 +2450,7 @@ type StatisticsResponse struct { func (x *StatisticsResponse) Reset() { *x = StatisticsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[34] + mi := &file_filer_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2314,7 +2463,7 @@ func (x *StatisticsResponse) String() string { func (*StatisticsResponse) ProtoMessage() {} func (x *StatisticsResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[34] + mi := &file_filer_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2327,7 +2476,7 @@ func (x *StatisticsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StatisticsResponse.ProtoReflect.Descriptor instead. func (*StatisticsResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{34} + return file_filer_proto_rawDescGZIP(), []int{36} } func (x *StatisticsResponse) GetTotalSize() uint64 { @@ -2351,6 +2500,124 @@ func (x *StatisticsResponse) GetFileCount() uint64 { return 0 } +type PingRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // default to ping itself + TargetType string `protobuf:"bytes,2,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` +} + +func (x *PingRequest) Reset() { + *x = PingRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PingRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PingRequest) ProtoMessage() {} + +func (x *PingRequest) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. +func (*PingRequest) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{37} +} + +func (x *PingRequest) GetTarget() string { + if x != nil { + return x.Target + } + return "" +} + +func (x *PingRequest) GetTargetType() string { + if x != nil { + return x.TargetType + } + return "" +} + +type PingResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StartTimeNs int64 `protobuf:"varint,1,opt,name=start_time_ns,json=startTimeNs,proto3" json:"start_time_ns,omitempty"` + RemoteTimeNs int64 `protobuf:"varint,2,opt,name=remote_time_ns,json=remoteTimeNs,proto3" json:"remote_time_ns,omitempty"` + StopTimeNs int64 `protobuf:"varint,3,opt,name=stop_time_ns,json=stopTimeNs,proto3" json:"stop_time_ns,omitempty"` +} + +func (x *PingResponse) Reset() { + *x = PingResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PingResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PingResponse) ProtoMessage() {} + +func (x *PingResponse) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[38] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. +func (*PingResponse) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{38} +} + +func (x *PingResponse) GetStartTimeNs() int64 { + if x != nil { + return x.StartTimeNs + } + return 0 +} + +func (x *PingResponse) GetRemoteTimeNs() int64 { + if x != nil { + return x.RemoteTimeNs + } + return 0 +} + +func (x *PingResponse) GetStopTimeNs() int64 { + if x != nil { + return x.StopTimeNs + } + return 0 +} + type GetFilerConfigurationRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2360,7 +2627,7 @@ type GetFilerConfigurationRequest struct { func (x *GetFilerConfigurationRequest) Reset() { *x = GetFilerConfigurationRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[35] + mi := &file_filer_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2373,7 +2640,7 @@ func (x *GetFilerConfigurationRequest) String() string { func (*GetFilerConfigurationRequest) ProtoMessage() {} func (x *GetFilerConfigurationRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[35] + mi := &file_filer_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2386,7 +2653,7 @@ func (x *GetFilerConfigurationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFilerConfigurationRequest.ProtoReflect.Descriptor instead. func (*GetFilerConfigurationRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{35} + return file_filer_proto_rawDescGZIP(), []int{39} } type GetFilerConfigurationResponse struct { @@ -2405,12 +2672,13 @@ type GetFilerConfigurationResponse struct { MetricsIntervalSec int32 `protobuf:"varint,10,opt,name=metrics_interval_sec,json=metricsIntervalSec,proto3" json:"metrics_interval_sec,omitempty"` Version string `protobuf:"bytes,11,opt,name=version,proto3" json:"version,omitempty"` ClusterId string `protobuf:"bytes,12,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + FilerGroup string `protobuf:"bytes,13,opt,name=filer_group,json=filerGroup,proto3" json:"filer_group,omitempty"` } func (x *GetFilerConfigurationResponse) Reset() { *x = GetFilerConfigurationResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[36] + mi := &file_filer_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2423,7 +2691,7 @@ func (x *GetFilerConfigurationResponse) String() string { func (*GetFilerConfigurationResponse) ProtoMessage() {} func (x *GetFilerConfigurationResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[36] + mi := &file_filer_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2436,7 +2704,7 @@ func (x *GetFilerConfigurationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFilerConfigurationResponse.ProtoReflect.Descriptor instead. func (*GetFilerConfigurationResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{36} + return file_filer_proto_rawDescGZIP(), []int{40} } func (x *GetFilerConfigurationResponse) GetMasters() []string { @@ -2516,21 +2784,31 @@ func (x *GetFilerConfigurationResponse) GetClusterId() string { return "" } +func (x *GetFilerConfigurationResponse) GetFilerGroup() string { + if x != nil { + return x.FilerGroup + } + return "" +} + type SubscribeMetadataRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName,proto3" json:"client_name,omitempty"` - PathPrefix string `protobuf:"bytes,2,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"` - SinceNs int64 `protobuf:"varint,3,opt,name=since_ns,json=sinceNs,proto3" json:"since_ns,omitempty"` - Signature int32 `protobuf:"varint,4,opt,name=signature,proto3" json:"signature,omitempty"` + ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName,proto3" json:"client_name,omitempty"` + PathPrefix string `protobuf:"bytes,2,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"` + SinceNs int64 `protobuf:"varint,3,opt,name=since_ns,json=sinceNs,proto3" json:"since_ns,omitempty"` + Signature int32 `protobuf:"varint,4,opt,name=signature,proto3" json:"signature,omitempty"` + PathPrefixes []string `protobuf:"bytes,6,rep,name=path_prefixes,json=pathPrefixes,proto3" json:"path_prefixes,omitempty"` + ClientId int32 `protobuf:"varint,7,opt,name=client_id,json=clientId,proto3" json:"client_id,omitempty"` + UntilNs int64 `protobuf:"varint,8,opt,name=until_ns,json=untilNs,proto3" json:"until_ns,omitempty"` } func (x *SubscribeMetadataRequest) Reset() { *x = SubscribeMetadataRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[37] + mi := &file_filer_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2543,7 +2821,7 @@ func (x *SubscribeMetadataRequest) String() string { func (*SubscribeMetadataRequest) ProtoMessage() {} func (x *SubscribeMetadataRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[37] + mi := &file_filer_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2556,7 +2834,7 @@ func (x *SubscribeMetadataRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SubscribeMetadataRequest.ProtoReflect.Descriptor instead. func (*SubscribeMetadataRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{37} + return file_filer_proto_rawDescGZIP(), []int{41} } func (x *SubscribeMetadataRequest) GetClientName() string { @@ -2587,6 +2865,27 @@ func (x *SubscribeMetadataRequest) GetSignature() int32 { return 0 } +func (x *SubscribeMetadataRequest) GetPathPrefixes() []string { + if x != nil { + return x.PathPrefixes + } + return nil +} + +func (x *SubscribeMetadataRequest) GetClientId() int32 { + if x != nil { + return x.ClientId + } + return 0 +} + +func (x *SubscribeMetadataRequest) GetUntilNs() int64 { + if x != nil { + return x.UntilNs + } + return 0 +} + type SubscribeMetadataResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2600,7 +2899,7 @@ type SubscribeMetadataResponse struct { func (x *SubscribeMetadataResponse) Reset() { *x = SubscribeMetadataResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[38] + mi := &file_filer_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2613,7 +2912,7 @@ func (x *SubscribeMetadataResponse) String() string { func (*SubscribeMetadataResponse) ProtoMessage() {} func (x *SubscribeMetadataResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[38] + mi := &file_filer_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2626,7 +2925,7 @@ func (x *SubscribeMetadataResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SubscribeMetadataResponse.ProtoReflect.Descriptor instead. func (*SubscribeMetadataResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{38} + return file_filer_proto_rawDescGZIP(), []int{42} } func (x *SubscribeMetadataResponse) GetDirectory() string { @@ -2663,7 +2962,7 @@ type LogEntry struct { func (x *LogEntry) Reset() { *x = LogEntry{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[39] + mi := &file_filer_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2676,7 +2975,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[39] + mi := &file_filer_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2689,7 +2988,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{39} + return file_filer_proto_rawDescGZIP(), []int{43} } func (x *LogEntry) GetTsNs() int64 { @@ -2726,7 +3025,7 @@ type KeepConnectedRequest struct { func (x *KeepConnectedRequest) Reset() { *x = KeepConnectedRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[40] + mi := &file_filer_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2739,7 +3038,7 @@ func (x *KeepConnectedRequest) String() string { func (*KeepConnectedRequest) ProtoMessage() {} func (x *KeepConnectedRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[40] + mi := &file_filer_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2752,7 +3051,7 @@ func (x *KeepConnectedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KeepConnectedRequest.ProtoReflect.Descriptor instead. func (*KeepConnectedRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{40} + return file_filer_proto_rawDescGZIP(), []int{44} } func (x *KeepConnectedRequest) GetName() string { @@ -2785,7 +3084,7 @@ type KeepConnectedResponse struct { func (x *KeepConnectedResponse) Reset() { *x = KeepConnectedResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[41] + mi := &file_filer_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2798,7 +3097,7 @@ func (x *KeepConnectedResponse) String() string { func (*KeepConnectedResponse) ProtoMessage() {} func (x *KeepConnectedResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[41] + mi := &file_filer_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2811,7 +3110,7 @@ func (x *KeepConnectedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KeepConnectedResponse.ProtoReflect.Descriptor instead. func (*KeepConnectedResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{41} + return file_filer_proto_rawDescGZIP(), []int{45} } type LocateBrokerRequest struct { @@ -2825,7 +3124,7 @@ type LocateBrokerRequest struct { func (x *LocateBrokerRequest) Reset() { *x = LocateBrokerRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[42] + mi := &file_filer_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2838,7 +3137,7 @@ func (x *LocateBrokerRequest) String() string { func (*LocateBrokerRequest) ProtoMessage() {} func (x *LocateBrokerRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[42] + mi := &file_filer_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2851,7 +3150,7 @@ func (x *LocateBrokerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerRequest.ProtoReflect.Descriptor instead. func (*LocateBrokerRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{42} + return file_filer_proto_rawDescGZIP(), []int{46} } func (x *LocateBrokerRequest) GetResource() string { @@ -2873,7 +3172,7 @@ type LocateBrokerResponse struct { func (x *LocateBrokerResponse) Reset() { *x = LocateBrokerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[43] + mi := &file_filer_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2886,7 +3185,7 @@ func (x *LocateBrokerResponse) String() string { func (*LocateBrokerResponse) ProtoMessage() {} func (x *LocateBrokerResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[43] + mi := &file_filer_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2899,7 +3198,7 @@ func (x *LocateBrokerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerResponse.ProtoReflect.Descriptor instead. func (*LocateBrokerResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{43} + return file_filer_proto_rawDescGZIP(), []int{47} } func (x *LocateBrokerResponse) GetFound() bool { @@ -2930,7 +3229,7 @@ type KvGetRequest struct { func (x *KvGetRequest) Reset() { *x = KvGetRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[44] + mi := &file_filer_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2943,7 +3242,7 @@ func (x *KvGetRequest) String() string { func (*KvGetRequest) ProtoMessage() {} func (x *KvGetRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[44] + mi := &file_filer_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2956,7 +3255,7 @@ func (x *KvGetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KvGetRequest.ProtoReflect.Descriptor instead. func (*KvGetRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{44} + return file_filer_proto_rawDescGZIP(), []int{48} } func (x *KvGetRequest) GetKey() []byte { @@ -2978,7 +3277,7 @@ type KvGetResponse struct { func (x *KvGetResponse) Reset() { *x = KvGetResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[45] + mi := &file_filer_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2991,7 +3290,7 @@ func (x *KvGetResponse) String() string { func (*KvGetResponse) ProtoMessage() {} func (x *KvGetResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[45] + mi := &file_filer_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3004,7 +3303,7 @@ func (x *KvGetResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KvGetResponse.ProtoReflect.Descriptor instead. func (*KvGetResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{45} + return file_filer_proto_rawDescGZIP(), []int{49} } func (x *KvGetResponse) GetValue() []byte { @@ -3033,7 +3332,7 @@ type KvPutRequest struct { func (x *KvPutRequest) Reset() { *x = KvPutRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[46] + mi := &file_filer_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3046,7 +3345,7 @@ func (x *KvPutRequest) String() string { func (*KvPutRequest) ProtoMessage() {} func (x *KvPutRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[46] + mi := &file_filer_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3059,7 +3358,7 @@ func (x *KvPutRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KvPutRequest.ProtoReflect.Descriptor instead. func (*KvPutRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{46} + return file_filer_proto_rawDescGZIP(), []int{50} } func (x *KvPutRequest) GetKey() []byte { @@ -3087,7 +3386,7 @@ type KvPutResponse struct { func (x *KvPutResponse) Reset() { *x = KvPutResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[47] + mi := &file_filer_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3100,7 +3399,7 @@ func (x *KvPutResponse) String() string { func (*KvPutResponse) ProtoMessage() {} func (x *KvPutResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[47] + mi := &file_filer_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3113,7 +3412,7 @@ func (x *KvPutResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KvPutResponse.ProtoReflect.Descriptor instead. func (*KvPutResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{47} + return file_filer_proto_rawDescGZIP(), []int{51} } func (x *KvPutResponse) GetError() string { @@ -3138,7 +3437,7 @@ type FilerConf struct { func (x *FilerConf) Reset() { *x = FilerConf{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[48] + mi := &file_filer_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3151,7 +3450,7 @@ func (x *FilerConf) String() string { func (*FilerConf) ProtoMessage() {} func (x *FilerConf) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[48] + mi := &file_filer_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3164,7 +3463,7 @@ func (x *FilerConf) ProtoReflect() protoreflect.Message { // Deprecated: Use FilerConf.ProtoReflect.Descriptor instead. func (*FilerConf) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{48} + return file_filer_proto_rawDescGZIP(), []int{52} } func (x *FilerConf) GetVersion() int32 { @@ -3184,7 +3483,7 @@ func (x *FilerConf) GetLocations() []*FilerConf_PathConf { ///////////////////////// // Remote Storage related ///////////////////////// -type DownloadToLocalRequest struct { +type CacheRemoteObjectToLocalClusterRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -3193,23 +3492,23 @@ type DownloadToLocalRequest struct { Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` } -func (x *DownloadToLocalRequest) Reset() { - *x = DownloadToLocalRequest{} +func (x *CacheRemoteObjectToLocalClusterRequest) Reset() { + *x = CacheRemoteObjectToLocalClusterRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[49] + mi := &file_filer_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *DownloadToLocalRequest) String() string { +func (x *CacheRemoteObjectToLocalClusterRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*DownloadToLocalRequest) ProtoMessage() {} +func (*CacheRemoteObjectToLocalClusterRequest) ProtoMessage() {} -func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[49] +func (x *CacheRemoteObjectToLocalClusterRequest) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3220,26 +3519,26 @@ func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use DownloadToLocalRequest.ProtoReflect.Descriptor instead. -func (*DownloadToLocalRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{49} +// Deprecated: Use CacheRemoteObjectToLocalClusterRequest.ProtoReflect.Descriptor instead. +func (*CacheRemoteObjectToLocalClusterRequest) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{53} } -func (x *DownloadToLocalRequest) GetDirectory() string { +func (x *CacheRemoteObjectToLocalClusterRequest) GetDirectory() string { if x != nil { return x.Directory } return "" } -func (x *DownloadToLocalRequest) GetName() string { +func (x *CacheRemoteObjectToLocalClusterRequest) GetName() string { if x != nil { return x.Name } return "" } -type DownloadToLocalResponse struct { +type CacheRemoteObjectToLocalClusterResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -3247,23 +3546,23 @@ type DownloadToLocalResponse struct { Entry *Entry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` } -func (x *DownloadToLocalResponse) Reset() { - *x = DownloadToLocalResponse{} +func (x *CacheRemoteObjectToLocalClusterResponse) Reset() { + *x = CacheRemoteObjectToLocalClusterResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[50] + mi := &file_filer_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *DownloadToLocalResponse) String() string { +func (x *CacheRemoteObjectToLocalClusterResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*DownloadToLocalResponse) ProtoMessage() {} +func (*CacheRemoteObjectToLocalClusterResponse) ProtoMessage() {} -func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[50] +func (x *CacheRemoteObjectToLocalClusterResponse) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3274,12 +3573,12 @@ func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use DownloadToLocalResponse.ProtoReflect.Descriptor instead. -func (*DownloadToLocalResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{50} +// Deprecated: Use CacheRemoteObjectToLocalClusterResponse.ProtoReflect.Descriptor instead. +func (*CacheRemoteObjectToLocalClusterResponse) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{54} } -func (x *DownloadToLocalResponse) GetEntry() *Entry { +func (x *CacheRemoteObjectToLocalClusterResponse) GetEntry() *Entry { if x != nil { return x.Entry } @@ -3300,7 +3599,7 @@ type LocateBrokerResponse_Resource struct { func (x *LocateBrokerResponse_Resource) Reset() { *x = LocateBrokerResponse_Resource{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[53] + mi := &file_filer_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3313,7 +3612,7 @@ func (x *LocateBrokerResponse_Resource) String() string { func (*LocateBrokerResponse_Resource) ProtoMessage() {} func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[53] + mi := &file_filer_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3326,7 +3625,7 @@ func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerResponse_Resource.ProtoReflect.Descriptor instead. func (*LocateBrokerResponse_Resource) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{43, 0} + return file_filer_proto_rawDescGZIP(), []int{47, 0} } func (x *LocateBrokerResponse_Resource) GetGrpcAddresses() string { @@ -3356,12 +3655,15 @@ type FilerConf_PathConf struct { Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"` VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"` ReadOnly bool `protobuf:"varint,8,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"` + DataCenter string `protobuf:"bytes,9,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` + Rack string `protobuf:"bytes,10,opt,name=rack,proto3" json:"rack,omitempty"` + DataNode string `protobuf:"bytes,11,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` } func (x *FilerConf_PathConf) Reset() { *x = FilerConf_PathConf{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[54] + mi := &file_filer_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3374,7 +3676,7 @@ func (x *FilerConf_PathConf) String() string { func (*FilerConf_PathConf) ProtoMessage() {} func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[54] + mi := &file_filer_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3387,7 +3689,7 @@ func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message { // Deprecated: Use FilerConf_PathConf.ProtoReflect.Descriptor instead. func (*FilerConf_PathConf) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{48, 0} + return file_filer_proto_rawDescGZIP(), []int{52, 0} } func (x *FilerConf_PathConf) GetLocationPrefix() string { @@ -3446,6 +3748,27 @@ func (x *FilerConf_PathConf) GetReadOnly() bool { return false } +func (x *FilerConf_PathConf) GetDataCenter() string { + if x != nil { + return x.DataCenter + } + return "" +} + +func (x *FilerConf_PathConf) GetRack() string { + if x != nil { + return x.Rack + } + return "" +} + +func (x *FilerConf_PathConf) GetDataNode() string { + if x != nil { + return x.DataNode + } + return "" +} + var File_filer_proto protoreflect.FileDescriptor var file_filer_proto_rawDesc = []byte{ @@ -3488,7 +3811,7 @@ var file_filer_proto_rawDesc = []byte{ 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xbf, 0x03, 0x0a, 0x05, 0x45, 0x6e, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xd5, 0x03, 0x0a, 0x05, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x69, @@ -3512,103 +3835,105 @@ var file_filer_proto_rawDesc = []byte{ 0x74, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x1a, 0x3b, - 0x0a, 0x0d, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x44, 0x0a, 0x09, 0x46, - 0x75, 0x6c, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x69, 0x72, 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, - 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, - 0x79, 0x22, 0x8f, 0x02, 0x0a, 0x11, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x09, 0x6f, 0x6c, 0x64, 0x5f, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6f, 0x6c, 0x64, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x2c, 0x0a, 0x09, 0x6e, 0x65, 0x77, 0x5f, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6e, 0x65, 0x77, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x63, 0x68, - 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x64, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x77, 0x5f, - 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x77, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, - 0x12, 0x31, 0x0a, 0x15, 0x69, 0x73, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6f, 0x74, 0x68, 0x65, - 0x72, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x12, 0x69, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x22, 0xe6, 0x02, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x13, 0x0a, 0x05, - 0x65, 0x5f, 0x74, 0x61, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x54, 0x61, - 0x67, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x03, 0x66, 0x69, 0x64, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x52, 0x03, 0x66, 0x69, 0x64, 0x12, 0x2f, 0x0a, 0x0a, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, - 0x64, 0x52, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x09, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x69, - 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, - 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x73, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x6d, 0x61, 0x6e, - 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x73, 0x43, - 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x22, 0x40, 0x0a, 0x11, - 0x46, 0x69, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, - 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0x58, - 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, - 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x07, - 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x22, 0x9d, 0x03, 0x0a, 0x0e, 0x46, 0x75, 0x73, - 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x66, - 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x66, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1b, - 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, - 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, - 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, - 0x16, 0x0a, 0x06, 0x63, 0x72, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x06, 0x63, 0x72, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x69, 0x6d, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x69, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, - 0x07, 0x74, 0x74, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, - 0x74, 0x74, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x5f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x79, 0x6d, 0x6c, - 0x69, 0x6e, 0x6b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x64, 0x35, - 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6d, 0x64, 0x35, 0x12, 0x1b, 0x0a, 0x09, 0x64, - 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc3, 0x01, 0x0a, 0x12, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x25, 0x0a, - 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x15, 0x0a, 0x06, 0x6f, 0x5f, 0x65, 0x78, 0x63, 0x6c, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x6f, 0x45, 0x78, 0x63, 0x6c, 0x12, 0x31, 0x0a, 0x15, 0x69, - 0x73, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x73, 0x46, 0x72, - 0x6f, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x1e, - 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x2b, + 0x79, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x71, 0x75, 0x6f, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x71, + 0x75, 0x6f, 0x74, 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x44, 0x0a, 0x09, 0x46, 0x75, 0x6c, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x69, 0x72, + 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x8f, 0x02, 0x0a, 0x11, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, + 0x09, 0x6f, 0x6c, 0x64, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x08, 0x6f, 0x6c, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x2c, 0x0a, 0x09, 0x6e, + 0x65, 0x77, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x08, 0x6e, 0x65, 0x77, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x26, + 0x0a, 0x0f, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x61, 0x74, + 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x77, 0x50, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x31, 0x0a, 0x15, 0x69, 0x73, 0x5f, 0x66, 0x72, 0x6f, + 0x6d, 0x5f, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x74, 0x68, + 0x65, 0x72, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0xe6, 0x02, 0x0a, 0x09, 0x46, 0x69, + 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, + 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6d, 0x74, 0x69, + 0x6d, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x65, 0x54, 0x61, 0x67, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x22, 0x0a, + 0x03, 0x66, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x52, 0x03, 0x66, 0x69, + 0x64, 0x12, 0x2f, 0x0a, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x64, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x52, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, + 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x4b, 0x65, + 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, + 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x43, 0x6f, 0x6d, 0x70, + 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x73, 0x5f, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0f, 0x69, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, + 0x73, 0x74, 0x22, 0x40, 0x0a, 0x11, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x73, 0x22, 0x58, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, + 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x07, 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x22, 0xe8, + 0x02, 0x0a, 0x0e, 0x46, 0x75, 0x73, 0x65, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6d, + 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x6f, 0x64, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, + 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x72, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x63, 0x72, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6d, 0x69, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x69, 0x6d, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x74, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x06, 0x74, 0x74, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, + 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, + 0x73, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, + 0x6b, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x6d, 0x64, 0x35, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6d, 0x64, 0x35, 0x12, + 0x12, 0x0a, 0x04, 0x72, 0x64, 0x65, 0x76, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, + 0x64, 0x65, 0x76, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x11, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x22, 0x82, 0x02, 0x0a, 0x12, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x25, + 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x15, 0x0a, 0x06, 0x6f, 0x5f, 0x65, 0x78, 0x63, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x6f, 0x45, 0x78, 0x63, 0x6c, 0x12, 0x31, 0x0a, 0x15, + 0x69, 0x73, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x73, 0x46, + 0x72, 0x6f, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, + 0x3d, 0x0a, 0x1b, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x70, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x73, 0x6b, 0x69, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x50, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x22, 0x2b, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xac, 0x01, 0x0a, 0x12, @@ -3668,328 +3993,393 @@ var file_filer_proto_rawDesc = []byte{ 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x1b, 0x0a, 0x19, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0xec, 0x01, 0x0a, 0x13, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1e, 0x0a, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x17, 0x0a, 0x07, 0x74, 0x74, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x06, 0x74, 0x74, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, - 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, - 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xe2, - 0x01, 0x0a, 0x14, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, - 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, - 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, - 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x22, 0x34, 0x0a, 0x13, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, 0x3d, 0x0a, 0x09, 0x4c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3b, 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, - 0x69, 0x63, 0x55, 0x72, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, - 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, - 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x54, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x20, 0x0a, 0x0a, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x7b, 0x0a, - 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4e, - 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x12, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x16, 0x43, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x39, 0x0a, 0x17, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1a, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x84, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, - 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, - 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, - 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x12, 0x53, 0x74, + 0xba, 0x01, 0x0a, 0x18, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, + 0x6f, 0x6c, 0x64, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x6c, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, + 0x6e, 0x65, 0x77, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x65, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, + 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x9a, 0x01, 0x0a, + 0x19, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x89, 0x02, 0x0a, 0x13, 0x41, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x74, 0x6c, + 0x5f, 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, 0x74, 0x6c, 0x53, + 0x65, 0x63, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, + 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xe1, 0x01, 0x0a, 0x14, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, + 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, + 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x08, 0x6c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x34, 0x0a, 0x13, 0x4c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, + 0x3d, 0x0a, 0x09, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x09, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x58, + 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, + 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, + 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, + 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xc3, 0x01, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x6d, + 0x61, 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x54, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x20, + 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x22, 0x7b, 0x0a, 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x5f, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, + 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, + 0x16, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, + 0x39, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1a, 0x0a, 0x18, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x84, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, + 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, + 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, + 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, + 0x12, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x46, + 0x0a, 0x0b, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x7a, 0x0a, 0x0c, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, + 0x12, 0x20, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x69, 0x6d, 0x65, + 0x4e, 0x73, 0x22, 0x1e, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0x9e, 0x03, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x20, + 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x61, 0x78, 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x6d, 0x61, 0x78, 0x4d, 0x62, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x5f, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x69, + 0x72, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x69, 0x70, 0x68, + 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x27, + 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x22, 0xf2, 0x01, 0x0a, 0x18, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, + 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, + 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, + 0x08, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x4e, 0x73, 0x22, 0x9a, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, + 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, + 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x61, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, + 0x17, 0x0a, 0x15, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x14, + 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, + 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x1a, 0x58, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x25, 0x0a, + 0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0c, 0x4b, + 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3b, 0x0a, + 0x0d, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x76, + 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xbd, 0x03, 0x0a, 0x09, 0x46, 0x69, + 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, + 0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xd9, 0x02, + 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, + 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, + 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, + 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, + 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, + 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x22, 0x5a, 0x0a, 0x26, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x27, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x32, 0x82, 0x0f, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, + 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, + 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, + 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, + 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1e, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, + 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, + 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x60, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, + 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, + 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, + 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, + 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, - 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x1e, 0x0a, 0x1c, 0x47, + 0x22, 0x00, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, + 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xfd, 0x02, 0x0a, 0x1d, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x61, 0x78, - 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x78, 0x4d, 0x62, - 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x69, 0x72, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x12, 0x30, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, - 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x95, 0x01, 0x0a, 0x18, - 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, - 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, - 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x69, - 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x22, 0x9a, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, - 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, - 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, - 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, - 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, - 0x22, 0x61, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x13, 0x0a, 0x05, - 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, - 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, - 0x65, 0x79, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x12, - 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x17, 0x0a, 0x15, 0x4b, 0x65, - 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, - 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x14, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, + 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, + 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, + 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, + 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x56, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, + 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, + 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x08, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, - 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0c, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3b, 0x0a, 0x0d, 0x4b, 0x76, 0x47, 0x65, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x25, 0x0a, - 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x22, 0xeb, 0x02, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x09, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x09, 0x6c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x87, 0x02, 0x0a, 0x08, 0x50, 0x61, 0x74, - 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1e, - 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, - 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, - 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x66, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x67, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, - 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, - 0x6c, 0x79, 0x22, 0x4a, 0x0a, 0x16, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x40, - 0x0a, 0x17, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, - 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x32, 0xb6, 0x0d, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, - 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, - 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, - 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, - 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, - 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, - 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, - 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, - 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, - 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, - 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, - 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, - 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, - 0x72, 0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, - 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x56, - 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, - 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, - 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, 0x65, 0x74, - 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x58, 0x0a, 0x0f, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x12, 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x6f, - 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, - 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, - 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, - 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, - 0x62, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, + 0x65, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, + 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x88, 0x01, 0x0a, 0x1f, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, + 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, + 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, + 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4004,70 +4394,74 @@ func file_filer_proto_rawDescGZIP() []byte { return file_filer_proto_rawDescData } -var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 55) +var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 59) var file_filer_proto_goTypes = []interface{}{ - (*LookupDirectoryEntryRequest)(nil), // 0: filer_pb.LookupDirectoryEntryRequest - (*LookupDirectoryEntryResponse)(nil), // 1: filer_pb.LookupDirectoryEntryResponse - (*ListEntriesRequest)(nil), // 2: filer_pb.ListEntriesRequest - (*ListEntriesResponse)(nil), // 3: filer_pb.ListEntriesResponse - (*RemoteEntry)(nil), // 4: filer_pb.RemoteEntry - (*Entry)(nil), // 5: filer_pb.Entry - (*FullEntry)(nil), // 6: filer_pb.FullEntry - (*EventNotification)(nil), // 7: filer_pb.EventNotification - (*FileChunk)(nil), // 8: filer_pb.FileChunk - (*FileChunkManifest)(nil), // 9: filer_pb.FileChunkManifest - (*FileId)(nil), // 10: filer_pb.FileId - (*FuseAttributes)(nil), // 11: filer_pb.FuseAttributes - (*CreateEntryRequest)(nil), // 12: filer_pb.CreateEntryRequest - (*CreateEntryResponse)(nil), // 13: filer_pb.CreateEntryResponse - (*UpdateEntryRequest)(nil), // 14: filer_pb.UpdateEntryRequest - (*UpdateEntryResponse)(nil), // 15: filer_pb.UpdateEntryResponse - (*AppendToEntryRequest)(nil), // 16: filer_pb.AppendToEntryRequest - (*AppendToEntryResponse)(nil), // 17: filer_pb.AppendToEntryResponse - (*DeleteEntryRequest)(nil), // 18: filer_pb.DeleteEntryRequest - (*DeleteEntryResponse)(nil), // 19: filer_pb.DeleteEntryResponse - (*AtomicRenameEntryRequest)(nil), // 20: filer_pb.AtomicRenameEntryRequest - (*AtomicRenameEntryResponse)(nil), // 21: filer_pb.AtomicRenameEntryResponse - (*AssignVolumeRequest)(nil), // 22: filer_pb.AssignVolumeRequest - (*AssignVolumeResponse)(nil), // 23: filer_pb.AssignVolumeResponse - (*LookupVolumeRequest)(nil), // 24: filer_pb.LookupVolumeRequest - (*Locations)(nil), // 25: filer_pb.Locations - (*Location)(nil), // 26: filer_pb.Location - (*LookupVolumeResponse)(nil), // 27: filer_pb.LookupVolumeResponse - (*Collection)(nil), // 28: filer_pb.Collection - (*CollectionListRequest)(nil), // 29: filer_pb.CollectionListRequest - (*CollectionListResponse)(nil), // 30: filer_pb.CollectionListResponse - (*DeleteCollectionRequest)(nil), // 31: filer_pb.DeleteCollectionRequest - (*DeleteCollectionResponse)(nil), // 32: filer_pb.DeleteCollectionResponse - (*StatisticsRequest)(nil), // 33: filer_pb.StatisticsRequest - (*StatisticsResponse)(nil), // 34: filer_pb.StatisticsResponse - (*GetFilerConfigurationRequest)(nil), // 35: filer_pb.GetFilerConfigurationRequest - (*GetFilerConfigurationResponse)(nil), // 36: filer_pb.GetFilerConfigurationResponse - (*SubscribeMetadataRequest)(nil), // 37: filer_pb.SubscribeMetadataRequest - (*SubscribeMetadataResponse)(nil), // 38: filer_pb.SubscribeMetadataResponse - (*LogEntry)(nil), // 39: filer_pb.LogEntry - (*KeepConnectedRequest)(nil), // 40: filer_pb.KeepConnectedRequest - (*KeepConnectedResponse)(nil), // 41: filer_pb.KeepConnectedResponse - (*LocateBrokerRequest)(nil), // 42: filer_pb.LocateBrokerRequest - (*LocateBrokerResponse)(nil), // 43: filer_pb.LocateBrokerResponse - (*KvGetRequest)(nil), // 44: filer_pb.KvGetRequest - (*KvGetResponse)(nil), // 45: filer_pb.KvGetResponse - (*KvPutRequest)(nil), // 46: filer_pb.KvPutRequest - (*KvPutResponse)(nil), // 47: filer_pb.KvPutResponse - (*FilerConf)(nil), // 48: filer_pb.FilerConf - (*DownloadToLocalRequest)(nil), // 49: filer_pb.DownloadToLocalRequest - (*DownloadToLocalResponse)(nil), // 50: filer_pb.DownloadToLocalResponse - nil, // 51: filer_pb.Entry.ExtendedEntry - nil, // 52: filer_pb.LookupVolumeResponse.LocationsMapEntry - (*LocateBrokerResponse_Resource)(nil), // 53: filer_pb.LocateBrokerResponse.Resource - (*FilerConf_PathConf)(nil), // 54: filer_pb.FilerConf.PathConf + (*LookupDirectoryEntryRequest)(nil), // 0: filer_pb.LookupDirectoryEntryRequest + (*LookupDirectoryEntryResponse)(nil), // 1: filer_pb.LookupDirectoryEntryResponse + (*ListEntriesRequest)(nil), // 2: filer_pb.ListEntriesRequest + (*ListEntriesResponse)(nil), // 3: filer_pb.ListEntriesResponse + (*RemoteEntry)(nil), // 4: filer_pb.RemoteEntry + (*Entry)(nil), // 5: filer_pb.Entry + (*FullEntry)(nil), // 6: filer_pb.FullEntry + (*EventNotification)(nil), // 7: filer_pb.EventNotification + (*FileChunk)(nil), // 8: filer_pb.FileChunk + (*FileChunkManifest)(nil), // 9: filer_pb.FileChunkManifest + (*FileId)(nil), // 10: filer_pb.FileId + (*FuseAttributes)(nil), // 11: filer_pb.FuseAttributes + (*CreateEntryRequest)(nil), // 12: filer_pb.CreateEntryRequest + (*CreateEntryResponse)(nil), // 13: filer_pb.CreateEntryResponse + (*UpdateEntryRequest)(nil), // 14: filer_pb.UpdateEntryRequest + (*UpdateEntryResponse)(nil), // 15: filer_pb.UpdateEntryResponse + (*AppendToEntryRequest)(nil), // 16: filer_pb.AppendToEntryRequest + (*AppendToEntryResponse)(nil), // 17: filer_pb.AppendToEntryResponse + (*DeleteEntryRequest)(nil), // 18: filer_pb.DeleteEntryRequest + (*DeleteEntryResponse)(nil), // 19: filer_pb.DeleteEntryResponse + (*AtomicRenameEntryRequest)(nil), // 20: filer_pb.AtomicRenameEntryRequest + (*AtomicRenameEntryResponse)(nil), // 21: filer_pb.AtomicRenameEntryResponse + (*StreamRenameEntryRequest)(nil), // 22: filer_pb.StreamRenameEntryRequest + (*StreamRenameEntryResponse)(nil), // 23: filer_pb.StreamRenameEntryResponse + (*AssignVolumeRequest)(nil), // 24: filer_pb.AssignVolumeRequest + (*AssignVolumeResponse)(nil), // 25: filer_pb.AssignVolumeResponse + (*LookupVolumeRequest)(nil), // 26: filer_pb.LookupVolumeRequest + (*Locations)(nil), // 27: filer_pb.Locations + (*Location)(nil), // 28: filer_pb.Location + (*LookupVolumeResponse)(nil), // 29: filer_pb.LookupVolumeResponse + (*Collection)(nil), // 30: filer_pb.Collection + (*CollectionListRequest)(nil), // 31: filer_pb.CollectionListRequest + (*CollectionListResponse)(nil), // 32: filer_pb.CollectionListResponse + (*DeleteCollectionRequest)(nil), // 33: filer_pb.DeleteCollectionRequest + (*DeleteCollectionResponse)(nil), // 34: filer_pb.DeleteCollectionResponse + (*StatisticsRequest)(nil), // 35: filer_pb.StatisticsRequest + (*StatisticsResponse)(nil), // 36: filer_pb.StatisticsResponse + (*PingRequest)(nil), // 37: filer_pb.PingRequest + (*PingResponse)(nil), // 38: filer_pb.PingResponse + (*GetFilerConfigurationRequest)(nil), // 39: filer_pb.GetFilerConfigurationRequest + (*GetFilerConfigurationResponse)(nil), // 40: filer_pb.GetFilerConfigurationResponse + (*SubscribeMetadataRequest)(nil), // 41: filer_pb.SubscribeMetadataRequest + (*SubscribeMetadataResponse)(nil), // 42: filer_pb.SubscribeMetadataResponse + (*LogEntry)(nil), // 43: filer_pb.LogEntry + (*KeepConnectedRequest)(nil), // 44: filer_pb.KeepConnectedRequest + (*KeepConnectedResponse)(nil), // 45: filer_pb.KeepConnectedResponse + (*LocateBrokerRequest)(nil), // 46: filer_pb.LocateBrokerRequest + (*LocateBrokerResponse)(nil), // 47: filer_pb.LocateBrokerResponse + (*KvGetRequest)(nil), // 48: filer_pb.KvGetRequest + (*KvGetResponse)(nil), // 49: filer_pb.KvGetResponse + (*KvPutRequest)(nil), // 50: filer_pb.KvPutRequest + (*KvPutResponse)(nil), // 51: filer_pb.KvPutResponse + (*FilerConf)(nil), // 52: filer_pb.FilerConf + (*CacheRemoteObjectToLocalClusterRequest)(nil), // 53: filer_pb.CacheRemoteObjectToLocalClusterRequest + (*CacheRemoteObjectToLocalClusterResponse)(nil), // 54: filer_pb.CacheRemoteObjectToLocalClusterResponse + nil, // 55: filer_pb.Entry.ExtendedEntry + nil, // 56: filer_pb.LookupVolumeResponse.LocationsMapEntry + (*LocateBrokerResponse_Resource)(nil), // 57: filer_pb.LocateBrokerResponse.Resource + (*FilerConf_PathConf)(nil), // 58: filer_pb.FilerConf.PathConf } var file_filer_proto_depIdxs = []int32{ 5, // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry 5, // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry 8, // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk 11, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes - 51, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry + 55, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry 4, // 5: filer_pb.Entry.remote_entry:type_name -> filer_pb.RemoteEntry 5, // 6: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry 5, // 7: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry @@ -4078,59 +4472,65 @@ var file_filer_proto_depIdxs = []int32{ 5, // 12: filer_pb.CreateEntryRequest.entry:type_name -> filer_pb.Entry 5, // 13: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry 8, // 14: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk - 26, // 15: filer_pb.Locations.locations:type_name -> filer_pb.Location - 52, // 16: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry - 28, // 17: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection - 7, // 18: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification - 53, // 19: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource - 54, // 20: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf - 5, // 21: filer_pb.DownloadToLocalResponse.entry:type_name -> filer_pb.Entry - 25, // 22: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations - 0, // 23: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest - 2, // 24: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest - 12, // 25: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest - 14, // 26: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest - 16, // 27: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest - 18, // 28: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest - 20, // 29: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest - 22, // 30: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest - 24, // 31: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest - 29, // 32: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest - 31, // 33: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest - 33, // 34: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest - 35, // 35: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest - 37, // 36: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest - 37, // 37: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest - 40, // 38: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest - 42, // 39: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest - 44, // 40: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest - 46, // 41: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest - 49, // 42: filer_pb.SeaweedFiler.DownloadToLocal:input_type -> filer_pb.DownloadToLocalRequest - 1, // 43: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse - 3, // 44: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse - 13, // 45: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse - 15, // 46: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse - 17, // 47: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse - 19, // 48: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse - 21, // 49: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse - 23, // 50: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse - 27, // 51: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse - 30, // 52: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse - 32, // 53: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse - 34, // 54: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse - 36, // 55: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse - 38, // 56: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse - 38, // 57: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse - 41, // 58: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse - 43, // 59: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse - 45, // 60: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse - 47, // 61: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse - 50, // 62: filer_pb.SeaweedFiler.DownloadToLocal:output_type -> filer_pb.DownloadToLocalResponse - 43, // [43:63] is the sub-list for method output_type - 23, // [23:43] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 7, // 15: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification + 28, // 16: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location + 28, // 17: filer_pb.Locations.locations:type_name -> filer_pb.Location + 56, // 18: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry + 30, // 19: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection + 7, // 20: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification + 57, // 21: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource + 58, // 22: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf + 5, // 23: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry + 27, // 24: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations + 0, // 25: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest + 2, // 26: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest + 12, // 27: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest + 14, // 28: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest + 16, // 29: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest + 18, // 30: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest + 20, // 31: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest + 22, // 32: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest + 24, // 33: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest + 26, // 34: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest + 31, // 35: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest + 33, // 36: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest + 35, // 37: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest + 37, // 38: filer_pb.SeaweedFiler.Ping:input_type -> filer_pb.PingRequest + 39, // 39: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest + 41, // 40: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest + 41, // 41: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest + 44, // 42: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest + 46, // 43: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest + 48, // 44: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest + 50, // 45: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest + 53, // 46: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest + 1, // 47: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse + 3, // 48: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse + 13, // 49: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse + 15, // 50: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse + 17, // 51: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse + 19, // 52: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse + 21, // 53: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse + 23, // 54: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse + 25, // 55: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse + 29, // 56: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse + 32, // 57: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse + 34, // 58: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse + 36, // 59: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse + 38, // 60: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse + 40, // 61: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse + 42, // 62: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse + 42, // 63: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse + 45, // 64: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse + 47, // 65: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse + 49, // 66: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse + 51, // 67: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse + 54, // 68: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse + 47, // [47:69] is the sub-list for method output_type + 25, // [25:47] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_filer_proto_init() } @@ -4404,7 +4804,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AssignVolumeRequest); i { + switch v := v.(*StreamRenameEntryRequest); i { case 0: return &v.state case 1: @@ -4416,7 +4816,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AssignVolumeResponse); i { + switch v := v.(*StreamRenameEntryResponse); i { case 0: return &v.state case 1: @@ -4428,7 +4828,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupVolumeRequest); i { + switch v := v.(*AssignVolumeRequest); i { case 0: return &v.state case 1: @@ -4440,7 +4840,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Locations); i { + switch v := v.(*AssignVolumeResponse); i { case 0: return &v.state case 1: @@ -4452,7 +4852,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Location); i { + switch v := v.(*LookupVolumeRequest); i { case 0: return &v.state case 1: @@ -4464,7 +4864,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupVolumeResponse); i { + switch v := v.(*Locations); i { case 0: return &v.state case 1: @@ -4476,7 +4876,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Collection); i { + switch v := v.(*Location); i { case 0: return &v.state case 1: @@ -4488,7 +4888,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionListRequest); i { + switch v := v.(*LookupVolumeResponse); i { case 0: return &v.state case 1: @@ -4500,7 +4900,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionListResponse); i { + switch v := v.(*Collection); i { case 0: return &v.state case 1: @@ -4512,7 +4912,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteCollectionRequest); i { + switch v := v.(*CollectionListRequest); i { case 0: return &v.state case 1: @@ -4524,7 +4924,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteCollectionResponse); i { + switch v := v.(*CollectionListResponse); i { case 0: return &v.state case 1: @@ -4536,7 +4936,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StatisticsRequest); i { + switch v := v.(*DeleteCollectionRequest); i { case 0: return &v.state case 1: @@ -4548,7 +4948,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StatisticsResponse); i { + switch v := v.(*DeleteCollectionResponse); i { case 0: return &v.state case 1: @@ -4560,7 +4960,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFilerConfigurationRequest); i { + switch v := v.(*StatisticsRequest); i { case 0: return &v.state case 1: @@ -4572,7 +4972,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFilerConfigurationResponse); i { + switch v := v.(*StatisticsResponse); i { case 0: return &v.state case 1: @@ -4584,7 +4984,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubscribeMetadataRequest); i { + switch v := v.(*PingRequest); i { case 0: return &v.state case 1: @@ -4596,7 +4996,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubscribeMetadataResponse); i { + switch v := v.(*PingResponse); i { case 0: return &v.state case 1: @@ -4608,7 +5008,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogEntry); i { + switch v := v.(*GetFilerConfigurationRequest); i { case 0: return &v.state case 1: @@ -4620,7 +5020,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeepConnectedRequest); i { + switch v := v.(*GetFilerConfigurationResponse); i { case 0: return &v.state case 1: @@ -4632,7 +5032,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeepConnectedResponse); i { + switch v := v.(*SubscribeMetadataRequest); i { case 0: return &v.state case 1: @@ -4644,7 +5044,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LocateBrokerRequest); i { + switch v := v.(*SubscribeMetadataResponse); i { case 0: return &v.state case 1: @@ -4656,7 +5056,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LocateBrokerResponse); i { + switch v := v.(*LogEntry); i { case 0: return &v.state case 1: @@ -4668,7 +5068,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvGetRequest); i { + switch v := v.(*KeepConnectedRequest); i { case 0: return &v.state case 1: @@ -4680,7 +5080,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvGetResponse); i { + switch v := v.(*KeepConnectedResponse); i { case 0: return &v.state case 1: @@ -4692,7 +5092,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvPutRequest); i { + switch v := v.(*LocateBrokerRequest); i { case 0: return &v.state case 1: @@ -4704,7 +5104,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvPutResponse); i { + switch v := v.(*LocateBrokerResponse); i { case 0: return &v.state case 1: @@ -4716,7 +5116,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FilerConf); i { + switch v := v.(*KvGetRequest); i { case 0: return &v.state case 1: @@ -4728,7 +5128,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DownloadToLocalRequest); i { + switch v := v.(*KvGetResponse); i { case 0: return &v.state case 1: @@ -4740,7 +5140,31 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DownloadToLocalResponse); i { + switch v := v.(*KvPutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*KvPutResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FilerConf); i { case 0: return &v.state case 1: @@ -4752,7 +5176,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LocateBrokerResponse_Resource); i { + switch v := v.(*CacheRemoteObjectToLocalClusterRequest); i { case 0: return &v.state case 1: @@ -4764,6 +5188,30 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CacheRemoteObjectToLocalClusterResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LocateBrokerResponse_Resource); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FilerConf_PathConf); i { case 0: return &v.state @@ -4782,7 +5230,7 @@ func file_filer_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_filer_proto_rawDesc, NumEnums: 0, - NumMessages: 55, + NumMessages: 59, NumExtensions: 0, NumServices: 1, }, @@ -4795,881 +5243,3 @@ func file_filer_proto_init() { file_filer_proto_goTypes = nil file_filer_proto_depIdxs = nil } - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// SeaweedFilerClient is the client API for SeaweedFiler service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SeaweedFilerClient interface { - LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) - ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error) - CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) - UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) - AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error) - DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) - AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) - AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) - LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) - CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) - DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) - Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) - GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) - SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) - SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) - KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) - LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) - KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error) - KvPut(ctx context.Context, in *KvPutRequest, opts ...grpc.CallOption) (*KvPutResponse, error) - DownloadToLocal(ctx context.Context, in *DownloadToLocalRequest, opts ...grpc.CallOption) (*DownloadToLocalResponse, error) -} - -type seaweedFilerClient struct { - cc grpc.ClientConnInterface -} - -func NewSeaweedFilerClient(cc grpc.ClientConnInterface) SeaweedFilerClient { - return &seaweedFilerClient{cc} -} - -func (c *seaweedFilerClient) LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) { - out := new(LookupDirectoryEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupDirectoryEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedFiler_serviceDesc.Streams[0], "/filer_pb.SeaweedFiler/ListEntries", opts...) - if err != nil { - return nil, err - } - x := &seaweedFilerListEntriesClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type SeaweedFiler_ListEntriesClient interface { - Recv() (*ListEntriesResponse, error) - grpc.ClientStream -} - -type seaweedFilerListEntriesClient struct { - grpc.ClientStream -} - -func (x *seaweedFilerListEntriesClient) Recv() (*ListEntriesResponse, error) { - m := new(ListEntriesResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) { - out := new(CreateEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CreateEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) { - out := new(UpdateEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error) { - out := new(AppendToEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AppendToEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) { - out := new(DeleteEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) { - out := new(AtomicRenameEntryResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AtomicRenameEntry", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) { - out := new(AssignVolumeResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) { - out := new(LookupVolumeResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) { - out := new(CollectionListResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CollectionList", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) { - out := new(DeleteCollectionResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteCollection", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { - out := new(StatisticsResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/Statistics", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) { - out := new(GetFilerConfigurationResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedFiler_serviceDesc.Streams[1], "/filer_pb.SeaweedFiler/SubscribeMetadata", opts...) - if err != nil { - return nil, err - } - x := &seaweedFilerSubscribeMetadataClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type SeaweedFiler_SubscribeMetadataClient interface { - Recv() (*SubscribeMetadataResponse, error) - grpc.ClientStream -} - -type seaweedFilerSubscribeMetadataClient struct { - grpc.ClientStream -} - -func (x *seaweedFilerSubscribeMetadataClient) Recv() (*SubscribeMetadataResponse, error) { - m := new(SubscribeMetadataResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedFilerClient) SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedFiler_serviceDesc.Streams[2], "/filer_pb.SeaweedFiler/SubscribeLocalMetadata", opts...) - if err != nil { - return nil, err - } - x := &seaweedFilerSubscribeLocalMetadataClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type SeaweedFiler_SubscribeLocalMetadataClient interface { - Recv() (*SubscribeMetadataResponse, error) - grpc.ClientStream -} - -type seaweedFilerSubscribeLocalMetadataClient struct { - grpc.ClientStream -} - -func (x *seaweedFilerSubscribeLocalMetadataClient) Recv() (*SubscribeMetadataResponse, error) { - m := new(SubscribeMetadataResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedFilerClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedFiler_serviceDesc.Streams[3], "/filer_pb.SeaweedFiler/KeepConnected", opts...) - if err != nil { - return nil, err - } - x := &seaweedFilerKeepConnectedClient{stream} - return x, nil -} - -type SeaweedFiler_KeepConnectedClient interface { - Send(*KeepConnectedRequest) error - Recv() (*KeepConnectedResponse, error) - grpc.ClientStream -} - -type seaweedFilerKeepConnectedClient struct { - grpc.ClientStream -} - -func (x *seaweedFilerKeepConnectedClient) Send(m *KeepConnectedRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *seaweedFilerKeepConnectedClient) Recv() (*KeepConnectedResponse, error) { - m := new(KeepConnectedResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedFilerClient) LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) { - out := new(LocateBrokerResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LocateBroker", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error) { - out := new(KvGetResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/KvGet", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) KvPut(ctx context.Context, in *KvPutRequest, opts ...grpc.CallOption) (*KvPutResponse, error) { - out := new(KvPutResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/KvPut", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedFilerClient) DownloadToLocal(ctx context.Context, in *DownloadToLocalRequest, opts ...grpc.CallOption) (*DownloadToLocalResponse, error) { - out := new(DownloadToLocalResponse) - err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DownloadToLocal", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SeaweedFilerServer is the server API for SeaweedFiler service. -type SeaweedFilerServer interface { - LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) - ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error - CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) - UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) - AppendToEntry(context.Context, *AppendToEntryRequest) (*AppendToEntryResponse, error) - DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) - AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error) - AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) - LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) - CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) - DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) - Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) - GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) - SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error - SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error - KeepConnected(SeaweedFiler_KeepConnectedServer) error - LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error) - KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error) - KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error) - DownloadToLocal(context.Context, *DownloadToLocalRequest) (*DownloadToLocalResponse, error) -} - -// UnimplementedSeaweedFilerServer can be embedded to have forward compatible implementations. -type UnimplementedSeaweedFilerServer struct { -} - -func (*UnimplementedSeaweedFilerServer) LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupDirectoryEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error { - return status.Errorf(codes.Unimplemented, "method ListEntries not implemented") -} -func (*UnimplementedSeaweedFilerServer) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CreateEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method UpdateEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) AppendToEntry(context.Context, *AppendToEntryRequest) (*AppendToEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AppendToEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AtomicRenameEntry not implemented") -} -func (*UnimplementedSeaweedFilerServer) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AssignVolume not implemented") -} -func (*UnimplementedSeaweedFilerServer) LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupVolume not implemented") -} -func (*UnimplementedSeaweedFilerServer) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CollectionList not implemented") -} -func (*UnimplementedSeaweedFilerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") -} -func (*UnimplementedSeaweedFilerServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") -} -func (*UnimplementedSeaweedFilerServer) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetFilerConfiguration not implemented") -} -func (*UnimplementedSeaweedFilerServer) SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error { - return status.Errorf(codes.Unimplemented, "method SubscribeMetadata not implemented") -} -func (*UnimplementedSeaweedFilerServer) SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error { - return status.Errorf(codes.Unimplemented, "method SubscribeLocalMetadata not implemented") -} -func (*UnimplementedSeaweedFilerServer) KeepConnected(SeaweedFiler_KeepConnectedServer) error { - return status.Errorf(codes.Unimplemented, "method KeepConnected not implemented") -} -func (*UnimplementedSeaweedFilerServer) LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LocateBroker not implemented") -} -func (*UnimplementedSeaweedFilerServer) KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method KvGet not implemented") -} -func (*UnimplementedSeaweedFilerServer) KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method KvPut not implemented") -} -func (*UnimplementedSeaweedFilerServer) DownloadToLocal(context.Context, *DownloadToLocalRequest) (*DownloadToLocalResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DownloadToLocal not implemented") -} - -func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) { - s.RegisterService(&_SeaweedFiler_serviceDesc, srv) -} - -func _SeaweedFiler_LookupDirectoryEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LookupDirectoryEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/LookupDirectoryEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, req.(*LookupDirectoryEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_ListEntries_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(ListEntriesRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(SeaweedFilerServer).ListEntries(m, &seaweedFilerListEntriesServer{stream}) -} - -type SeaweedFiler_ListEntriesServer interface { - Send(*ListEntriesResponse) error - grpc.ServerStream -} - -type seaweedFilerListEntriesServer struct { - grpc.ServerStream -} - -func (x *seaweedFilerListEntriesServer) Send(m *ListEntriesResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CreateEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).CreateEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/CreateEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).CreateEntry(ctx, req.(*CreateEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(UpdateEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).UpdateEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_AppendToEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AppendToEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).AppendToEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/AppendToEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).AppendToEntry(ctx, req.(*AppendToEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).DeleteEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/DeleteEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).DeleteEntry(ctx, req.(*DeleteEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_AtomicRenameEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AtomicRenameEntryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/AtomicRenameEntry", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, req.(*AtomicRenameEntryRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AssignVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).AssignVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/AssignVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).AssignVolume(ctx, req.(*AssignVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_LookupVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LookupVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).LookupVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/LookupVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).LookupVolume(ctx, req.(*LookupVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_CollectionList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CollectionListRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).CollectionList(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/CollectionList", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).CollectionList(ctx, req.(*CollectionListRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_DeleteCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteCollectionRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).DeleteCollection(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/DeleteCollection", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).DeleteCollection(ctx, req.(*DeleteCollectionRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StatisticsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).Statistics(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/Statistics", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).Statistics(ctx, req.(*StatisticsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFilerConfigurationRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/GetFilerConfiguration", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, req.(*GetFilerConfigurationRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_SubscribeMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(SubscribeMetadataRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(SeaweedFilerServer).SubscribeMetadata(m, &seaweedFilerSubscribeMetadataServer{stream}) -} - -type SeaweedFiler_SubscribeMetadataServer interface { - Send(*SubscribeMetadataResponse) error - grpc.ServerStream -} - -type seaweedFilerSubscribeMetadataServer struct { - grpc.ServerStream -} - -func (x *seaweedFilerSubscribeMetadataServer) Send(m *SubscribeMetadataResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _SeaweedFiler_SubscribeLocalMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(SubscribeMetadataRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(SeaweedFilerServer).SubscribeLocalMetadata(m, &seaweedFilerSubscribeLocalMetadataServer{stream}) -} - -type SeaweedFiler_SubscribeLocalMetadataServer interface { - Send(*SubscribeMetadataResponse) error - grpc.ServerStream -} - -type seaweedFilerSubscribeLocalMetadataServer struct { - grpc.ServerStream -} - -func (x *seaweedFilerSubscribeLocalMetadataServer) Send(m *SubscribeMetadataResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _SeaweedFiler_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SeaweedFilerServer).KeepConnected(&seaweedFilerKeepConnectedServer{stream}) -} - -type SeaweedFiler_KeepConnectedServer interface { - Send(*KeepConnectedResponse) error - Recv() (*KeepConnectedRequest, error) - grpc.ServerStream -} - -type seaweedFilerKeepConnectedServer struct { - grpc.ServerStream -} - -func (x *seaweedFilerKeepConnectedServer) Send(m *KeepConnectedResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *seaweedFilerKeepConnectedServer) Recv() (*KeepConnectedRequest, error) { - m := new(KeepConnectedRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _SeaweedFiler_LocateBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LocateBrokerRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).LocateBroker(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/LocateBroker", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).LocateBroker(ctx, req.(*LocateBrokerRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_KvGet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(KvGetRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).KvGet(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/KvGet", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).KvGet(ctx, req.(*KvGetRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_KvPut_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(KvPutRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).KvPut(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/KvPut", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).KvPut(ctx, req.(*KvPutRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedFiler_DownloadToLocal_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DownloadToLocalRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).DownloadToLocal(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/DownloadToLocal", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).DownloadToLocal(ctx, req.(*DownloadToLocalRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ - ServiceName: "filer_pb.SeaweedFiler", - HandlerType: (*SeaweedFilerServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "LookupDirectoryEntry", - Handler: _SeaweedFiler_LookupDirectoryEntry_Handler, - }, - { - MethodName: "CreateEntry", - Handler: _SeaweedFiler_CreateEntry_Handler, - }, - { - MethodName: "UpdateEntry", - Handler: _SeaweedFiler_UpdateEntry_Handler, - }, - { - MethodName: "AppendToEntry", - Handler: _SeaweedFiler_AppendToEntry_Handler, - }, - { - MethodName: "DeleteEntry", - Handler: _SeaweedFiler_DeleteEntry_Handler, - }, - { - MethodName: "AtomicRenameEntry", - Handler: _SeaweedFiler_AtomicRenameEntry_Handler, - }, - { - MethodName: "AssignVolume", - Handler: _SeaweedFiler_AssignVolume_Handler, - }, - { - MethodName: "LookupVolume", - Handler: _SeaweedFiler_LookupVolume_Handler, - }, - { - MethodName: "CollectionList", - Handler: _SeaweedFiler_CollectionList_Handler, - }, - { - MethodName: "DeleteCollection", - Handler: _SeaweedFiler_DeleteCollection_Handler, - }, - { - MethodName: "Statistics", - Handler: _SeaweedFiler_Statistics_Handler, - }, - { - MethodName: "GetFilerConfiguration", - Handler: _SeaweedFiler_GetFilerConfiguration_Handler, - }, - { - MethodName: "LocateBroker", - Handler: _SeaweedFiler_LocateBroker_Handler, - }, - { - MethodName: "KvGet", - Handler: _SeaweedFiler_KvGet_Handler, - }, - { - MethodName: "KvPut", - Handler: _SeaweedFiler_KvPut_Handler, - }, - { - MethodName: "DownloadToLocal", - Handler: _SeaweedFiler_DownloadToLocal_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "ListEntries", - Handler: _SeaweedFiler_ListEntries_Handler, - ServerStreams: true, - }, - { - StreamName: "SubscribeMetadata", - Handler: _SeaweedFiler_SubscribeMetadata_Handler, - ServerStreams: true, - }, - { - StreamName: "SubscribeLocalMetadata", - Handler: _SeaweedFiler_SubscribeLocalMetadata_Handler, - ServerStreams: true, - }, - { - StreamName: "KeepConnected", - Handler: _SeaweedFiler_KeepConnected_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "filer.proto", -} diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index b34bcabb8..266bdae51 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -20,7 +20,7 @@ var ( ) type FilerClient interface { - WithFilerClient(fn func(SeaweedFilerClient) error) error + WithFilerClient(streamingMode bool, fn func(SeaweedFilerClient) error) error AdjustedUrl(location *Location) string } @@ -28,7 +28,7 @@ func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry dir, name := fullFilePath.DirAndName() - err = filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + err = filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { request := &LookupDirectoryEntryRequest{ Directory: dir, @@ -38,9 +38,6 @@ func GetEntry(filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry // glog.V(3).Infof("read %s request: %v", fullFilePath, request) resp, err := LookupEntry(client, request) if err != nil { - if err == ErrNotFound { - return nil - } glog.V(3).Infof("read %s %v: %v", fullFilePath, resp, err) return err } @@ -86,13 +83,13 @@ func ReadDirAllEntries(filerClient FilerClient, fullDirPath util.FullPath, prefi } func List(filerClient FilerClient, parentDirectoryPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { return doSeaweedList(client, util.FullPath(parentDirectoryPath), prefix, fn, startFrom, inclusive, limit) }) } func doList(filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { return doSeaweedList(client, fullDirPath, prefix, fn, startFrom, inclusive, limit) }) } @@ -104,9 +101,13 @@ func SeaweedList(client SeaweedFilerClient, parentDirectoryPath, prefix string, func doSeaweedList(client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) { // Redundancy limit to make it correctly judge whether it is the last file. redLimit := limit - if limit != math.MaxInt32 && limit != 0 { + + if limit < math.MaxInt32 && limit != 0 { redLimit = limit + 1 } + if redLimit > math.MaxInt32 { + redLimit = math.MaxInt32 + } request := &ListEntriesRequest{ Directory: string(fullDirPath), Prefix: prefix, @@ -156,7 +157,7 @@ func doSeaweedList(client SeaweedFilerClient, fullDirPath util.FullPath, prefix func Exists(filerClient FilerClient, parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) { - err = filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + err = filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { request := &LookupDirectoryEntryRequest{ Directory: parentDirectoryPath, @@ -184,7 +185,7 @@ func Exists(filerClient FilerClient, parentDirectoryPath string, entryName strin func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string, entry *Entry) (err error) { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { request := &UpdateEntryRequest{ Directory: parentDirectoryPath, @@ -203,7 +204,7 @@ func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string } func Mkdir(filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { return DoMkdir(client, parentDirectoryPath, dirName, fn) }) } @@ -240,7 +241,7 @@ func DoMkdir(client SeaweedFilerClient, parentDirectoryPath string, dirName stri } func MkFile(filerClient FilerClient, parentDirectoryPath string, fileName string, chunks []*FileChunk, fn func(entry *Entry)) error { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { entry := &Entry{ Name: fileName, @@ -275,7 +276,7 @@ func MkFile(filerClient FilerClient, parentDirectoryPath string, fileName string } func Remove(filerClient FilerClient, parentDirectoryPath, name string, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster bool, signatures []int32) error { - return filerClient.WithFilerClient(func(client SeaweedFilerClient) error { + return filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error { return DoRemove(client, parentDirectoryPath, name, isDeleteData, isRecursive, ignoreRecursiveErr, isFromOtherCluster, signatures) }) } diff --git a/weed/pb/filer_pb/filer_grpc.pb.go b/weed/pb/filer_pb/filer_grpc.pb.go new file mode 100644 index 000000000..4a5f47d71 --- /dev/null +++ b/weed/pb/filer_pb/filer_grpc.pb.go @@ -0,0 +1,998 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package filer_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedFilerClient is the client API for SeaweedFiler service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedFilerClient interface { + LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) + ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error) + CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) + UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) + AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error) + DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) + AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) + StreamRenameEntry(ctx context.Context, in *StreamRenameEntryRequest, opts ...grpc.CallOption) (SeaweedFiler_StreamRenameEntryClient, error) + AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) + LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) + CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) + DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) + Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) + Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) + GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) + SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) + SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) + KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) + LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) + KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error) + KvPut(ctx context.Context, in *KvPutRequest, opts ...grpc.CallOption) (*KvPutResponse, error) + CacheRemoteObjectToLocalCluster(ctx context.Context, in *CacheRemoteObjectToLocalClusterRequest, opts ...grpc.CallOption) (*CacheRemoteObjectToLocalClusterResponse, error) +} + +type seaweedFilerClient struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedFilerClient(cc grpc.ClientConnInterface) SeaweedFilerClient { + return &seaweedFilerClient{cc} +} + +func (c *seaweedFilerClient) LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) { + out := new(LookupDirectoryEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupDirectoryEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[0], "/filer_pb.SeaweedFiler/ListEntries", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerListEntriesClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type SeaweedFiler_ListEntriesClient interface { + Recv() (*ListEntriesResponse, error) + grpc.ClientStream +} + +type seaweedFilerListEntriesClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerListEntriesClient) Recv() (*ListEntriesResponse, error) { + m := new(ListEntriesResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) { + out := new(CreateEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CreateEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) { + out := new(UpdateEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error) { + out := new(AppendToEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AppendToEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) { + out := new(DeleteEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) { + out := new(AtomicRenameEntryResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AtomicRenameEntry", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) StreamRenameEntry(ctx context.Context, in *StreamRenameEntryRequest, opts ...grpc.CallOption) (SeaweedFiler_StreamRenameEntryClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[1], "/filer_pb.SeaweedFiler/StreamRenameEntry", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerStreamRenameEntryClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type SeaweedFiler_StreamRenameEntryClient interface { + Recv() (*StreamRenameEntryResponse, error) + grpc.ClientStream +} + +type seaweedFilerStreamRenameEntryClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerStreamRenameEntryClient) Recv() (*StreamRenameEntryResponse, error) { + m := new(StreamRenameEntryResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) { + out := new(AssignVolumeResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) { + out := new(LookupVolumeResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) { + out := new(CollectionListResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CollectionList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) { + out := new(DeleteCollectionResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteCollection", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { + out := new(StatisticsResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/Statistics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { + out := new(PingResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) { + out := new(GetFilerConfigurationResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[2], "/filer_pb.SeaweedFiler/SubscribeMetadata", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerSubscribeMetadataClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type SeaweedFiler_SubscribeMetadataClient interface { + Recv() (*SubscribeMetadataResponse, error) + grpc.ClientStream +} + +type seaweedFilerSubscribeMetadataClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerSubscribeMetadataClient) Recv() (*SubscribeMetadataResponse, error) { + m := new(SubscribeMetadataResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedFilerClient) SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[3], "/filer_pb.SeaweedFiler/SubscribeLocalMetadata", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerSubscribeLocalMetadataClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type SeaweedFiler_SubscribeLocalMetadataClient interface { + Recv() (*SubscribeMetadataResponse, error) + grpc.ClientStream +} + +type seaweedFilerSubscribeLocalMetadataClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerSubscribeLocalMetadataClient) Recv() (*SubscribeMetadataResponse, error) { + m := new(SubscribeMetadataResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedFilerClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[4], "/filer_pb.SeaweedFiler/KeepConnected", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerKeepConnectedClient{stream} + return x, nil +} + +type SeaweedFiler_KeepConnectedClient interface { + Send(*KeepConnectedRequest) error + Recv() (*KeepConnectedResponse, error) + grpc.ClientStream +} + +type seaweedFilerKeepConnectedClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerKeepConnectedClient) Send(m *KeepConnectedRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedFilerKeepConnectedClient) Recv() (*KeepConnectedResponse, error) { + m := new(KeepConnectedResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedFilerClient) LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) { + out := new(LocateBrokerResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/LocateBroker", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error) { + out := new(KvGetResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/KvGet", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) KvPut(ctx context.Context, in *KvPutRequest, opts ...grpc.CallOption) (*KvPutResponse, error) { + out := new(KvPutResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/KvPut", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) CacheRemoteObjectToLocalCluster(ctx context.Context, in *CacheRemoteObjectToLocalClusterRequest, opts ...grpc.CallOption) (*CacheRemoteObjectToLocalClusterResponse, error) { + out := new(CacheRemoteObjectToLocalClusterResponse) + err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CacheRemoteObjectToLocalCluster", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SeaweedFilerServer is the server API for SeaweedFiler service. +// All implementations must embed UnimplementedSeaweedFilerServer +// for forward compatibility +type SeaweedFilerServer interface { + LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) + ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error + CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) + UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) + AppendToEntry(context.Context, *AppendToEntryRequest) (*AppendToEntryResponse, error) + DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) + AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error) + StreamRenameEntry(*StreamRenameEntryRequest, SeaweedFiler_StreamRenameEntryServer) error + AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) + LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) + CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) + DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) + Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) + Ping(context.Context, *PingRequest) (*PingResponse, error) + GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) + SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error + SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error + KeepConnected(SeaweedFiler_KeepConnectedServer) error + LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error) + KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error) + KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error) + CacheRemoteObjectToLocalCluster(context.Context, *CacheRemoteObjectToLocalClusterRequest) (*CacheRemoteObjectToLocalClusterResponse, error) + mustEmbedUnimplementedSeaweedFilerServer() +} + +// UnimplementedSeaweedFilerServer must be embedded to have forward compatible implementations. +type UnimplementedSeaweedFilerServer struct { +} + +func (UnimplementedSeaweedFilerServer) LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LookupDirectoryEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error { + return status.Errorf(codes.Unimplemented, "method ListEntries not implemented") +} +func (UnimplementedSeaweedFilerServer) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) AppendToEntry(context.Context, *AppendToEntryRequest) (*AppendToEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AppendToEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AtomicRenameEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) StreamRenameEntry(*StreamRenameEntryRequest, SeaweedFiler_StreamRenameEntryServer) error { + return status.Errorf(codes.Unimplemented, "method StreamRenameEntry not implemented") +} +func (UnimplementedSeaweedFilerServer) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AssignVolume not implemented") +} +func (UnimplementedSeaweedFilerServer) LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LookupVolume not implemented") +} +func (UnimplementedSeaweedFilerServer) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CollectionList not implemented") +} +func (UnimplementedSeaweedFilerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") +} +func (UnimplementedSeaweedFilerServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") +} +func (UnimplementedSeaweedFilerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedSeaweedFilerServer) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFilerConfiguration not implemented") +} +func (UnimplementedSeaweedFilerServer) SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error { + return status.Errorf(codes.Unimplemented, "method SubscribeMetadata not implemented") +} +func (UnimplementedSeaweedFilerServer) SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error { + return status.Errorf(codes.Unimplemented, "method SubscribeLocalMetadata not implemented") +} +func (UnimplementedSeaweedFilerServer) KeepConnected(SeaweedFiler_KeepConnectedServer) error { + return status.Errorf(codes.Unimplemented, "method KeepConnected not implemented") +} +func (UnimplementedSeaweedFilerServer) LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LocateBroker not implemented") +} +func (UnimplementedSeaweedFilerServer) KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method KvGet not implemented") +} +func (UnimplementedSeaweedFilerServer) KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method KvPut not implemented") +} +func (UnimplementedSeaweedFilerServer) CacheRemoteObjectToLocalCluster(context.Context, *CacheRemoteObjectToLocalClusterRequest) (*CacheRemoteObjectToLocalClusterResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CacheRemoteObjectToLocalCluster not implemented") +} +func (UnimplementedSeaweedFilerServer) mustEmbedUnimplementedSeaweedFilerServer() {} + +// UnsafeSeaweedFilerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedFilerServer will +// result in compilation errors. +type UnsafeSeaweedFilerServer interface { + mustEmbedUnimplementedSeaweedFilerServer() +} + +func RegisterSeaweedFilerServer(s grpc.ServiceRegistrar, srv SeaweedFilerServer) { + s.RegisterService(&SeaweedFiler_ServiceDesc, srv) +} + +func _SeaweedFiler_LookupDirectoryEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupDirectoryEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/LookupDirectoryEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).LookupDirectoryEntry(ctx, req.(*LookupDirectoryEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_ListEntries_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ListEntriesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SeaweedFilerServer).ListEntries(m, &seaweedFilerListEntriesServer{stream}) +} + +type SeaweedFiler_ListEntriesServer interface { + Send(*ListEntriesResponse) error + grpc.ServerStream +} + +type seaweedFilerListEntriesServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerListEntriesServer) Send(m *ListEntriesResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).CreateEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/CreateEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).CreateEntry(ctx, req.(*CreateEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).UpdateEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_AppendToEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AppendToEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).AppendToEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/AppendToEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).AppendToEntry(ctx, req.(*AppendToEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).DeleteEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/DeleteEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).DeleteEntry(ctx, req.(*DeleteEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_AtomicRenameEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AtomicRenameEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/AtomicRenameEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, req.(*AtomicRenameEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_StreamRenameEntry_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StreamRenameEntryRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SeaweedFilerServer).StreamRenameEntry(m, &seaweedFilerStreamRenameEntryServer{stream}) +} + +type SeaweedFiler_StreamRenameEntryServer interface { + Send(*StreamRenameEntryResponse) error + grpc.ServerStream +} + +type seaweedFilerStreamRenameEntryServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerStreamRenameEntryServer) Send(m *StreamRenameEntryResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AssignVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).AssignVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/AssignVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).AssignVolume(ctx, req.(*AssignVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_LookupVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).LookupVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/LookupVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).LookupVolume(ctx, req.(*LookupVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_CollectionList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CollectionListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).CollectionList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/CollectionList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).CollectionList(ctx, req.(*CollectionListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_DeleteCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteCollectionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).DeleteCollection(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/DeleteCollection", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).DeleteCollection(ctx, req.(*DeleteCollectionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).Statistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/Statistics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).Statistics(ctx, req.(*StatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).Ping(ctx, req.(*PingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetFilerConfigurationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/GetFilerConfiguration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, req.(*GetFilerConfigurationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_SubscribeMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeMetadataRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SeaweedFilerServer).SubscribeMetadata(m, &seaweedFilerSubscribeMetadataServer{stream}) +} + +type SeaweedFiler_SubscribeMetadataServer interface { + Send(*SubscribeMetadataResponse) error + grpc.ServerStream +} + +type seaweedFilerSubscribeMetadataServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerSubscribeMetadataServer) Send(m *SubscribeMetadataResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _SeaweedFiler_SubscribeLocalMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeMetadataRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SeaweedFilerServer).SubscribeLocalMetadata(m, &seaweedFilerSubscribeLocalMetadataServer{stream}) +} + +type SeaweedFiler_SubscribeLocalMetadataServer interface { + Send(*SubscribeMetadataResponse) error + grpc.ServerStream +} + +type seaweedFilerSubscribeLocalMetadataServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerSubscribeLocalMetadataServer) Send(m *SubscribeMetadataResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _SeaweedFiler_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedFilerServer).KeepConnected(&seaweedFilerKeepConnectedServer{stream}) +} + +type SeaweedFiler_KeepConnectedServer interface { + Send(*KeepConnectedResponse) error + Recv() (*KeepConnectedRequest, error) + grpc.ServerStream +} + +type seaweedFilerKeepConnectedServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerKeepConnectedServer) Send(m *KeepConnectedResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedFilerKeepConnectedServer) Recv() (*KeepConnectedRequest, error) { + m := new(KeepConnectedRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _SeaweedFiler_LocateBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LocateBrokerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).LocateBroker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/LocateBroker", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).LocateBroker(ctx, req.(*LocateBrokerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_KvGet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(KvGetRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).KvGet(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/KvGet", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).KvGet(ctx, req.(*KvGetRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_KvPut_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(KvPutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).KvPut(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/KvPut", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).KvPut(ctx, req.(*KvPutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_CacheRemoteObjectToLocalCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CacheRemoteObjectToLocalClusterRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).CacheRemoteObjectToLocalCluster(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/CacheRemoteObjectToLocalCluster", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).CacheRemoteObjectToLocalCluster(ctx, req.(*CacheRemoteObjectToLocalClusterRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SeaweedFiler_ServiceDesc is the grpc.ServiceDesc for SeaweedFiler service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "filer_pb.SeaweedFiler", + HandlerType: (*SeaweedFilerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "LookupDirectoryEntry", + Handler: _SeaweedFiler_LookupDirectoryEntry_Handler, + }, + { + MethodName: "CreateEntry", + Handler: _SeaweedFiler_CreateEntry_Handler, + }, + { + MethodName: "UpdateEntry", + Handler: _SeaweedFiler_UpdateEntry_Handler, + }, + { + MethodName: "AppendToEntry", + Handler: _SeaweedFiler_AppendToEntry_Handler, + }, + { + MethodName: "DeleteEntry", + Handler: _SeaweedFiler_DeleteEntry_Handler, + }, + { + MethodName: "AtomicRenameEntry", + Handler: _SeaweedFiler_AtomicRenameEntry_Handler, + }, + { + MethodName: "AssignVolume", + Handler: _SeaweedFiler_AssignVolume_Handler, + }, + { + MethodName: "LookupVolume", + Handler: _SeaweedFiler_LookupVolume_Handler, + }, + { + MethodName: "CollectionList", + Handler: _SeaweedFiler_CollectionList_Handler, + }, + { + MethodName: "DeleteCollection", + Handler: _SeaweedFiler_DeleteCollection_Handler, + }, + { + MethodName: "Statistics", + Handler: _SeaweedFiler_Statistics_Handler, + }, + { + MethodName: "Ping", + Handler: _SeaweedFiler_Ping_Handler, + }, + { + MethodName: "GetFilerConfiguration", + Handler: _SeaweedFiler_GetFilerConfiguration_Handler, + }, + { + MethodName: "LocateBroker", + Handler: _SeaweedFiler_LocateBroker_Handler, + }, + { + MethodName: "KvGet", + Handler: _SeaweedFiler_KvGet_Handler, + }, + { + MethodName: "KvPut", + Handler: _SeaweedFiler_KvPut_Handler, + }, + { + MethodName: "CacheRemoteObjectToLocalCluster", + Handler: _SeaweedFiler_CacheRemoteObjectToLocalCluster_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ListEntries", + Handler: _SeaweedFiler_ListEntries_Handler, + ServerStreams: true, + }, + { + StreamName: "StreamRenameEntry", + Handler: _SeaweedFiler_StreamRenameEntry_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeMetadata", + Handler: _SeaweedFiler_SubscribeMetadata_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeLocalMetadata", + Handler: _SeaweedFiler_SubscribeLocalMetadata_Handler, + ServerStreams: true, + }, + { + StreamName: "KeepConnected", + Handler: _SeaweedFiler_KeepConnected_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "filer.proto", +} diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 186a5e20a..2c9526b80 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "github.com/chrislusf/seaweedfs/weed/glog" @@ -16,6 +17,17 @@ func (entry *Entry) IsInRemoteOnly() bool { return len(entry.Chunks) == 0 && entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0 } +func (entry *Entry) IsDirectoryKeyObject() bool { + return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != "" +} + +func (entry *Entry) FileMode() (fileMode os.FileMode) { + if entry != nil && entry.Attributes != nil { + fileMode = os.FileMode(entry.Attributes.FileMode) + } + return +} + func ToFileIdObject(fileIdStr string) (*FileId, error) { t, err := needle.ParseFileIdFromString(fileIdStr) if err != nil { @@ -128,13 +140,17 @@ func LookupEntry(client SeaweedFilerClient, request *LookupDirectoryEntryRequest var ErrNotFound = errors.New("filer: no entry is found in filer store") +func IsEmpty(event *SubscribeMetadataResponse) bool { + return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry == nil +} func IsCreate(event *SubscribeMetadataResponse) bool { return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry == nil } func IsUpdate(event *SubscribeMetadataResponse) bool { return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry != nil && - event.Directory == event.EventNotification.NewParentPath + event.Directory == event.EventNotification.NewParentPath && + event.EventNotification.NewEntry.Name == event.EventNotification.OldEntry.Name } func IsDelete(event *SubscribeMetadataResponse) bool { return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry != nil @@ -142,7 +158,8 @@ func IsDelete(event *SubscribeMetadataResponse) bool { func IsRename(event *SubscribeMetadataResponse) bool { return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry != nil && - event.Directory != event.EventNotification.NewParentPath + (event.Directory != event.EventNotification.NewParentPath || + event.EventNotification.NewEntry.Name != event.EventNotification.OldEntry.Name) } var _ = ptrie.KeyProvider(&FilerConf_PathConf{}) diff --git a/weed/pb/filer_pb/signature.go b/weed/pb/filer_pb/signature.go deleted file mode 100644 index e13afc656..000000000 --- a/weed/pb/filer_pb/signature.go +++ /dev/null @@ -1,13 +0,0 @@ -package filer_pb - -func (r *CreateEntryRequest) AddSignature(sig int32) { - r.Signatures = append(r.Signatures, sig) -} -func (r *CreateEntryRequest) HasSigned(sig int32) bool { - for _, s := range r.Signatures { - if s == sig { - return true - } - } - return false -} diff --git a/weed/pb/filer_pb_tail.go b/weed/pb/filer_pb_tail.go index 31fb62fb3..2d6a7898b 100644 --- a/weed/pb/filer_pb_tail.go +++ b/weed/pb/filer_pb_tail.go @@ -5,19 +5,28 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" "google.golang.org/grpc" "io" "time" ) +type EventErrorType int + +const ( + TrivialOnError EventErrorType = iota + FatalOnError + RetryForeverOnError +) + type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error -func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption, - clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, - processEventFn ProcessMetadataFunc, fatalOnError bool) error { +func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, clientName string, clientId int32, + pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, untilTsNs int64, selfSignature int32, + processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error { - err := WithFilerClient(filerAddress, grpcDialOption, makeFunc( - clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError)) + err := WithFilerClient(true, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(clientName, clientId, + pathPrefix, additionalPathPrefixes, &lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType)) if err != nil { return fmt.Errorf("subscribing filer meta change: %v", err) } @@ -25,11 +34,11 @@ func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption, } func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, - clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, - processEventFn ProcessMetadataFunc, fatalOnError bool) error { + clientName string, clientId int32, pathPrefix string, lastTsNs *int64, untilTsNs int64, selfSignature int32, + processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error { - err := filerClient.WithFilerClient(makeFunc( - clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError)) + err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(clientName, clientId, + pathPrefix, nil, lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType)) if err != nil { return fmt.Errorf("subscribing filer meta change: %v", err) } @@ -37,16 +46,19 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, return nil } -func makeFunc(clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, - processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error { +func makeSubscribeMetadataFunc(clientName string, clientId int32, pathPrefix string, additionalPathPrefixes []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, + processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) func(client filer_pb.SeaweedFilerClient) error { return func(client filer_pb.SeaweedFilerClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ - ClientName: clientName, - PathPrefix: pathPrefix, - SinceNs: lastTsNs, - Signature: selfSignature, + ClientName: clientName, + PathPrefix: pathPrefix, + PathPrefixes: additionalPathPrefixes, + SinceNs: *lastTsNs, + Signature: selfSignature, + ClientId: clientId, + UntilNs: untilTsNs, }) if err != nil { return fmt.Errorf("subscribe: %v", err) @@ -62,13 +74,23 @@ func makeFunc(clientName string, pathPrefix string, lastTsNs int64, selfSignatur } if err := processEventFn(resp); err != nil { - if fatalOnError { + switch eventErrorType { + case TrivialOnError: + glog.Errorf("process %v: %v", resp, err) + case FatalOnError: glog.Fatalf("process %v: %v", resp, err) - } else { + case RetryForeverOnError: + util.RetryForever("followMetaUpdates", func() error { + return processEventFn(resp) + }, func(err error) bool { + glog.Errorf("process %v: %v", resp, err) + return true + }) + default: glog.Errorf("process %v: %v", resp, err) } } - lastTsNs = resp.TsNs + *lastTsNs = resp.TsNs } } } @@ -82,11 +104,11 @@ func AddOffsetFunc(processEventFn ProcessMetadataFunc, offsetInterval time.Durat } counter++ if lastWriteTime.Add(offsetInterval).Before(time.Now()) { - counter = 0 lastWriteTime = time.Now() if err := offsetFunc(counter, resp.TsNs); err != nil { return err } + counter = 0 } return nil } diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 8543df366..990cf74f9 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/util" "math/rand" "net/http" "strconv" @@ -96,7 +98,8 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG return existingConnection, nil } - grpcConnection, err := GrpcDial(context.Background(), address, opts...) + ctx := context.Background() + grpcConnection, err := GrpcDial(ctx, address, opts...) if err != nil { return nil, fmt.Errorf("fail to dial %s: %v", address, err) } @@ -111,42 +114,42 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG return vgc, nil } -func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error { +// WithGrpcClient In streamingMode, always use a fresh connection. Otherwise, try to reuse an existing connection. +func WithGrpcClient(streamingMode bool, fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error { - vgc, err := getOrCreateConnection(address, opts...) - if err != nil { - return fmt.Errorf("getOrCreateConnection %s: %v", address, err) - } - executionErr := fn(vgc.ClientConn) - if executionErr != nil { - if strings.Contains(executionErr.Error(), "transport") || - strings.Contains(executionErr.Error(), "connection closed") { - grpcClientsLock.Lock() - if t, ok := grpcClients[address]; ok { - if t.version == vgc.version { - vgc.Close() - delete(grpcClients, address) + if !streamingMode { + vgc, err := getOrCreateConnection(address, opts...) + if err != nil { + return fmt.Errorf("getOrCreateConnection %s: %v", address, err) + } + executionErr := fn(vgc.ClientConn) + if executionErr != nil { + if strings.Contains(executionErr.Error(), "transport") || + strings.Contains(executionErr.Error(), "connection closed") { + grpcClientsLock.Lock() + if t, ok := grpcClients[address]; ok { + if t.version == vgc.version { + vgc.Close() + delete(grpcClients, address) + } } + grpcClientsLock.Unlock() } - grpcClientsLock.Unlock() } - } - - return executionErr -} - -func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) { - return ParseServerAddress(server, 10000) -} -func ParseServersToGrpcAddresses(servers []string) (serverGrpcAddresses []string, err error) { - for _, server := range servers { - if serverGrpcAddress, parseErr := ParseServerToGrpcAddress(server); parseErr == nil { - serverGrpcAddresses = append(serverGrpcAddresses, serverGrpcAddress) - } else { - return nil, parseErr + return executionErr + } else { + grpcConnection, err := GrpcDial(context.Background(), address, opts...) + if err != nil { + return fmt.Errorf("fail to dial %s: %v", address, err) } + defer grpcConnection.Close() + executionErr := fn(grpcConnection) + if executionErr != nil { + return executionErr + } + return nil } - return + } func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) { @@ -158,7 +161,7 @@ func ParseServerAddress(server string, deltaPort int) (newServerAddress string, newPort := int(port) + deltaPort - return fmt.Sprintf("%s:%d", host, newPort), nil + return util.JoinHostPort(host, newPort), nil } func hostAndPort(address string) (host string, port uint64, err error) { @@ -183,7 +186,7 @@ func ServerToGrpcAddress(server string) (serverGrpcAddress string) { grpcPort := int(port) + 10000 - return fmt.Sprintf("%s:%d", host, grpcPort) + return util.JoinHostPort(host, grpcPort) } func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) { @@ -194,30 +197,32 @@ func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) { port := int(grpcPort) - 10000 - return fmt.Sprintf("%s:%d", host, port) + return util.JoinHostPort(host, port) } -func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error { +func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error { + return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { + client := master_pb.NewSeaweedClient(grpcConnection) + return fn(client) + }, master.ToGrpcAddress(), grpcDialOption) - masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master) - if parseErr != nil { - return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr) - } +} - return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { - client := master_pb.NewSeaweedClient(grpcConnection) +func WithVolumeServerClient(streamingMode bool, volumeServer ServerAddress, grpcDialOption grpc.DialOption, fn func(client volume_server_pb.VolumeServerClient) error) error { + return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { + client := volume_server_pb.NewVolumeServerClient(grpcConnection) return fn(client) - }, masterGrpcAddress, grpcDialOption) + }, volumeServer.ToGrpcAddress(), grpcDialOption) } -func WithOneOfGrpcMasterClients(masterGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) { +func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) { for _, masterGrpcAddress := range masterGrpcAddresses { - err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := master_pb.NewSeaweedClient(grpcConnection) return fn(client) - }, masterGrpcAddress, grpcDialOption) + }, masterGrpcAddress.ToGrpcAddress(), grpcDialOption) if err == nil { return nil } @@ -226,42 +231,37 @@ func WithOneOfGrpcMasterClients(masterGrpcAddresses []string, grpcDialOption grp return err } -func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error { +func WithBrokerGrpcClient(streamingMode bool, brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error { - return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := messaging_pb.NewSeaweedMessagingClient(grpcConnection) return fn(client) }, brokerGrpcAddress, grpcDialOption) } -func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error { - - filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer) - if parseErr != nil { - return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr) - } +func WithFilerClient(streamingMode bool, filer ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error { - return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn) + return WithGrpcFilerClient(streamingMode, filer, grpcDialOption, fn) } -func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error { +func WithGrpcFilerClient(streamingMode bool, filerGrpcAddress ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error { - return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) - }, filerGrpcAddress, grpcDialOption) + }, filerGrpcAddress.ToGrpcAddress(), grpcDialOption) } -func WithOneOfGrpcFilerClients(filerGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) { +func WithOneOfGrpcFilerClients(streamingMode bool, filerAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) { - for _, filerGrpcAddress := range filerGrpcAddresses { - err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + for _, filerAddress := range filerAddresses { + err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) - }, filerGrpcAddress, grpcDialOption) + }, filerAddress.ToGrpcAddress(), grpcDialOption) if err == nil { return nil } diff --git a/weed/pb/iam_pb/iam.pb.go b/weed/pb/iam_pb/iam.pb.go index 7d0b6281b..89a4f1584 100644 --- a/weed/pb/iam_pb/iam.pb.go +++ b/weed/pb/iam_pb/iam.pb.go @@ -1,15 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: iam.proto package iam_pb import ( - context "context" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -23,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type S3ApiConfiguration struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -312,45 +305,3 @@ func file_iam_proto_init() { file_iam_proto_goTypes = nil file_iam_proto_depIdxs = nil } - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SeaweedIdentityAccessManagementClient interface { -} - -type seaweedIdentityAccessManagementClient struct { - cc grpc.ClientConnInterface -} - -func NewSeaweedIdentityAccessManagementClient(cc grpc.ClientConnInterface) SeaweedIdentityAccessManagementClient { - return &seaweedIdentityAccessManagementClient{cc} -} - -// SeaweedIdentityAccessManagementServer is the server API for SeaweedIdentityAccessManagement service. -type SeaweedIdentityAccessManagementServer interface { -} - -// UnimplementedSeaweedIdentityAccessManagementServer can be embedded to have forward compatible implementations. -type UnimplementedSeaweedIdentityAccessManagementServer struct { -} - -func RegisterSeaweedIdentityAccessManagementServer(s *grpc.Server, srv SeaweedIdentityAccessManagementServer) { - s.RegisterService(&_SeaweedIdentityAccessManagement_serviceDesc, srv) -} - -var _SeaweedIdentityAccessManagement_serviceDesc = grpc.ServiceDesc{ - ServiceName: "iam_pb.SeaweedIdentityAccessManagement", - HandlerType: (*SeaweedIdentityAccessManagementServer)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{}, - Metadata: "iam.proto", -} diff --git a/weed/pb/iam_pb/iam_grpc.pb.go b/weed/pb/iam_pb/iam_grpc.pb.go new file mode 100644 index 000000000..b9438a295 --- /dev/null +++ b/weed/pb/iam_pb/iam_grpc.pb.go @@ -0,0 +1,62 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package iam_pb + +import ( + grpc "google.golang.org/grpc" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedIdentityAccessManagementClient interface { +} + +type seaweedIdentityAccessManagementClient struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedIdentityAccessManagementClient(cc grpc.ClientConnInterface) SeaweedIdentityAccessManagementClient { + return &seaweedIdentityAccessManagementClient{cc} +} + +// SeaweedIdentityAccessManagementServer is the server API for SeaweedIdentityAccessManagement service. +// All implementations must embed UnimplementedSeaweedIdentityAccessManagementServer +// for forward compatibility +type SeaweedIdentityAccessManagementServer interface { + mustEmbedUnimplementedSeaweedIdentityAccessManagementServer() +} + +// UnimplementedSeaweedIdentityAccessManagementServer must be embedded to have forward compatible implementations. +type UnimplementedSeaweedIdentityAccessManagementServer struct { +} + +func (UnimplementedSeaweedIdentityAccessManagementServer) mustEmbedUnimplementedSeaweedIdentityAccessManagementServer() { +} + +// UnsafeSeaweedIdentityAccessManagementServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedIdentityAccessManagementServer will +// result in compilation errors. +type UnsafeSeaweedIdentityAccessManagementServer interface { + mustEmbedUnimplementedSeaweedIdentityAccessManagementServer() +} + +func RegisterSeaweedIdentityAccessManagementServer(s grpc.ServiceRegistrar, srv SeaweedIdentityAccessManagementServer) { + s.RegisterService(&SeaweedIdentityAccessManagement_ServiceDesc, srv) +} + +// SeaweedIdentityAccessManagement_ServiceDesc is the grpc.ServiceDesc for SeaweedIdentityAccessManagement service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SeaweedIdentityAccessManagement_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "iam_pb.SeaweedIdentityAccessManagement", + HandlerType: (*SeaweedIdentityAccessManagementServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{}, + Metadata: "iam.proto", +} diff --git a/weed/pb/master.proto b/weed/pb/master.proto index fcd5cebf1..84ec7374b 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -7,206 +7,235 @@ option go_package = "github.com/chrislusf/seaweedfs/weed/pb/master_pb"; ////////////////////////////////////////////////// service Seaweed { - rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { - } - rpc KeepConnected (stream KeepConnectedRequest) returns (stream VolumeLocation) { - } - rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { - } - rpc Assign (AssignRequest) returns (AssignResponse) { - } - rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { - } - rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { - } - rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) { - } - rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) { - } - rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) { - } - rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) { - } - rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) { - } - rpc ListMasterClients (ListMasterClientsRequest) returns (ListMasterClientsResponse) { - } - rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) { - } - rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) { - } - + rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { + } + rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { + } + rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { + } + rpc Assign (AssignRequest) returns (AssignResponse) { + } + rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { + } + rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { + } + rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) { + } + rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) { + } + rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) { + } + rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) { + } + rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) { + } + rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) { + } + rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) { + } + rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) { + } + rpc Ping (PingRequest) returns (PingResponse) { + } + rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) { + } + rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) { + } + rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) { + } } ////////////////////////////////////////////////// message Heartbeat { - string ip = 1; - uint32 port = 2; - string public_url = 3; - uint64 max_file_key = 5; - string data_center = 6; - string rack = 7; - uint32 admin_port = 8; - repeated VolumeInformationMessage volumes = 9; - // delta volumes - repeated VolumeShortInformationMessage new_volumes = 10; - repeated VolumeShortInformationMessage deleted_volumes = 11; - bool has_no_volumes = 12; - - // erasure coding - repeated VolumeEcShardInformationMessage ec_shards = 16; - // delta erasure coding shards - repeated VolumeEcShardInformationMessage new_ec_shards = 17; - repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; - bool has_no_ec_shards = 19; - - map max_volume_counts = 4; - + string ip = 1; + uint32 port = 2; + string public_url = 3; + uint64 max_file_key = 5; + string data_center = 6; + string rack = 7; + uint32 admin_port = 8; + repeated VolumeInformationMessage volumes = 9; + // delta volumes + repeated VolumeShortInformationMessage new_volumes = 10; + repeated VolumeShortInformationMessage deleted_volumes = 11; + bool has_no_volumes = 12; + + // erasure coding + repeated VolumeEcShardInformationMessage ec_shards = 16; + // delta erasure coding shards + repeated VolumeEcShardInformationMessage new_ec_shards = 17; + repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; + bool has_no_ec_shards = 19; + + map max_volume_counts = 4; + uint32 grpc_port = 20; + repeated string location_uuids = 21; } message HeartbeatResponse { - uint64 volume_size_limit = 1; - string leader = 2; - string metrics_address = 3; - uint32 metrics_interval_seconds = 4; - repeated StorageBackend storage_backends = 5; + uint64 volume_size_limit = 1; + string leader = 2; + string metrics_address = 3; + uint32 metrics_interval_seconds = 4; + repeated StorageBackend storage_backends = 5; + repeated string duplicated_uuids = 6; } message VolumeInformationMessage { - uint32 id = 1; - uint64 size = 2; - string collection = 3; - uint64 file_count = 4; - uint64 delete_count = 5; - uint64 deleted_byte_count = 6; - bool read_only = 7; - uint32 replica_placement = 8; - uint32 version = 9; - uint32 ttl = 10; - uint32 compact_revision = 11; - int64 modified_at_second = 12; - string remote_storage_name = 13; - string remote_storage_key = 14; - string disk_type = 15; + uint32 id = 1; + uint64 size = 2; + string collection = 3; + uint64 file_count = 4; + uint64 delete_count = 5; + uint64 deleted_byte_count = 6; + bool read_only = 7; + uint32 replica_placement = 8; + uint32 version = 9; + uint32 ttl = 10; + uint32 compact_revision = 11; + int64 modified_at_second = 12; + string remote_storage_name = 13; + string remote_storage_key = 14; + string disk_type = 15; } message VolumeShortInformationMessage { - uint32 id = 1; - string collection = 3; - uint32 replica_placement = 8; - uint32 version = 9; - uint32 ttl = 10; - string disk_type = 15; + uint32 id = 1; + string collection = 3; + uint32 replica_placement = 8; + uint32 version = 9; + uint32 ttl = 10; + string disk_type = 15; } message VolumeEcShardInformationMessage { - uint32 id = 1; - string collection = 2; - uint32 ec_index_bits = 3; - string disk_type = 4; + uint32 id = 1; + string collection = 2; + uint32 ec_index_bits = 3; + string disk_type = 4; } message StorageBackend { - string type = 1; - string id = 2; - map properties = 3; + string type = 1; + string id = 2; + map properties = 3; } message Empty { } message SuperBlockExtra { - message ErasureCoding { - uint32 data = 1; - uint32 parity = 2; - repeated uint32 volume_ids = 3; - } - ErasureCoding erasure_coding = 1; + message ErasureCoding { + uint32 data = 1; + uint32 parity = 2; + repeated uint32 volume_ids = 3; + } + ErasureCoding erasure_coding = 1; } message KeepConnectedRequest { - string name = 1; - uint32 grpc_port = 2; + string client_type = 1; + string client_address = 3; + string version = 4; + string filer_group = 5; } message VolumeLocation { - string url = 1; - string public_url = 2; - repeated uint32 new_vids = 3; - repeated uint32 deleted_vids = 4; - string leader = 5; // optional when leader is not itself - string data_center = 6; // optional when DataCenter is in use + string url = 1; + string public_url = 2; + repeated uint32 new_vids = 3; + repeated uint32 deleted_vids = 4; + string leader = 5; // optional when leader is not itself + string data_center = 6; // optional when DataCenter is in use + uint32 grpc_port = 7; + repeated uint32 new_ec_vids = 8; + repeated uint32 deleted_ec_vids = 9; +} + +message ClusterNodeUpdate { + string node_type = 1; + string address = 2; + bool is_leader = 3; + bool is_add = 4; + string filer_group = 5; + int64 created_at_ns = 6; +} + +message KeepConnectedResponse { + VolumeLocation volume_location = 1; + ClusterNodeUpdate cluster_node_update = 2; } message LookupVolumeRequest { - repeated string volume_or_file_ids = 1; - string collection = 2; // optional, a bit faster if provided. + repeated string volume_or_file_ids = 1; + string collection = 2; // optional, a bit faster if provided. } message LookupVolumeResponse { - message VolumeIdLocation { - string volume_or_file_id = 1; - repeated Location locations = 2; - string error = 3; - string auth = 4; - } - repeated VolumeIdLocation volume_id_locations = 1; + message VolumeIdLocation { + string volume_or_file_id = 1; + repeated Location locations = 2; + string error = 3; + string auth = 4; + } + repeated VolumeIdLocation volume_id_locations = 1; } message Location { - string url = 1; - string public_url = 2; + string url = 1; + string public_url = 2; + uint32 grpc_port = 3; } message AssignRequest { - uint64 count = 1; - string replication = 2; - string collection = 3; - string ttl = 4; - string data_center = 5; - string rack = 6; - string data_node = 7; - uint32 memory_map_max_size_mb = 8; - uint32 Writable_volume_count = 9; - string disk_type = 10; + uint64 count = 1; + string replication = 2; + string collection = 3; + string ttl = 4; + string data_center = 5; + string rack = 6; + string data_node = 7; + uint32 memory_map_max_size_mb = 8; + uint32 Writable_volume_count = 9; + string disk_type = 10; } message AssignResponse { - string fid = 1; - string url = 2; - string public_url = 3; - uint64 count = 4; - string error = 5; - string auth = 6; + string fid = 1; + uint64 count = 4; + string error = 5; + string auth = 6; + repeated Location replicas = 7; + Location location = 8; } message StatisticsRequest { - string replication = 1; - string collection = 2; - string ttl = 3; - string disk_type = 4; + string replication = 1; + string collection = 2; + string ttl = 3; + string disk_type = 4; } message StatisticsResponse { - uint64 total_size = 4; - uint64 used_size = 5; - uint64 file_count = 6; + uint64 total_size = 4; + uint64 used_size = 5; + uint64 file_count = 6; } // // collection related // message Collection { - string name = 1; + string name = 1; } message CollectionListRequest { - bool include_normal_volumes = 1; - bool include_ec_volumes = 2; + bool include_normal_volumes = 1; + bool include_ec_volumes = 2; } message CollectionListResponse { - repeated Collection collections = 1; + repeated Collection collections = 1; } message CollectionDeleteRequest { - string name = 1; + string name = 1; } message CollectionDeleteResponse { } @@ -215,55 +244,58 @@ message CollectionDeleteResponse { // volume related // message DiskInfo { - string type = 1; - int64 volume_count = 2; - int64 max_volume_count = 3; - int64 free_volume_count = 4; - int64 active_volume_count = 5; - repeated VolumeInformationMessage volume_infos = 6; - repeated VolumeEcShardInformationMessage ec_shard_infos = 7; - int64 remote_volume_count = 8; + string type = 1; + int64 volume_count = 2; + int64 max_volume_count = 3; + int64 free_volume_count = 4; + int64 active_volume_count = 5; + repeated VolumeInformationMessage volume_infos = 6; + repeated VolumeEcShardInformationMessage ec_shard_infos = 7; + int64 remote_volume_count = 8; } message DataNodeInfo { - string id = 1; - map diskInfos = 2; + string id = 1; + map diskInfos = 2; + uint32 grpc_port = 3; } message RackInfo { - string id = 1; - repeated DataNodeInfo data_node_infos = 2; - map diskInfos = 3; + string id = 1; + repeated DataNodeInfo data_node_infos = 2; + map diskInfos = 3; } message DataCenterInfo { - string id = 1; - repeated RackInfo rack_infos = 2; - map diskInfos = 3; + string id = 1; + repeated RackInfo rack_infos = 2; + map diskInfos = 3; } message TopologyInfo { - string id = 1; - repeated DataCenterInfo data_center_infos = 2; - map diskInfos = 3; + string id = 1; + repeated DataCenterInfo data_center_infos = 2; + map diskInfos = 3; } message VolumeListRequest { } message VolumeListResponse { - TopologyInfo topology_info = 1; - uint64 volume_size_limit_mb = 2; + TopologyInfo topology_info = 1; + uint64 volume_size_limit_mb = 2; } message LookupEcVolumeRequest { - uint32 volume_id = 1; + uint32 volume_id = 1; } message LookupEcVolumeResponse { - uint32 volume_id = 1; - message EcShardIdLocation { - uint32 shard_id = 1; - repeated Location locations = 2; - } - repeated EcShardIdLocation shard_id_locations = 2; + uint32 volume_id = 1; + message EcShardIdLocation { + uint32 shard_id = 1; + repeated Location locations = 2; + } + repeated EcShardIdLocation shard_id_locations = 2; } message VacuumVolumeRequest { - float garbage_threshold = 1; + float garbage_threshold = 1; + uint32 volume_id = 2; + string collection = 3; } message VacuumVolumeResponse { } @@ -271,37 +303,81 @@ message VacuumVolumeResponse { message GetMasterConfigurationRequest { } message GetMasterConfigurationResponse { - string metrics_address = 1; - uint32 metrics_interval_seconds = 2; - repeated StorageBackend storage_backends = 3; - string default_replication = 4; - string leader = 5; - uint32 volume_size_limit_m_b = 6; - bool volume_preallocate = 7; + string metrics_address = 1; + uint32 metrics_interval_seconds = 2; + repeated StorageBackend storage_backends = 3; + string default_replication = 4; + string leader = 5; + uint32 volume_size_limit_m_b = 6; + bool volume_preallocate = 7; } -message ListMasterClientsRequest { - string client_type = 1; +message ListClusterNodesRequest { + string client_type = 1; + string filer_group = 2; } -message ListMasterClientsResponse { - repeated string grpc_addresses = 1; +message ListClusterNodesResponse { + message ClusterNode { + string address = 1; + string version = 2; + bool is_leader = 3; + int64 created_at_ns = 4; + } + repeated ClusterNode cluster_nodes = 1; } message LeaseAdminTokenRequest { - int64 previous_token = 1; - int64 previous_lock_time = 2; - string lock_name = 3; - string client_name = 4; + int64 previous_token = 1; + int64 previous_lock_time = 2; + string lock_name = 3; + string client_name = 4; + string message = 5; } message LeaseAdminTokenResponse { - int64 token = 1; - int64 lock_ts_ns = 2; + int64 token = 1; + int64 lock_ts_ns = 2; } message ReleaseAdminTokenRequest { - int64 previous_token = 1; - int64 previous_lock_time = 2; - string lock_name = 3; + int64 previous_token = 1; + int64 previous_lock_time = 2; + string lock_name = 3; } message ReleaseAdminTokenResponse { } + +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +} + +message RaftAddServerRequest { + string id = 1; + string address = 2; + bool voter = 3; +} +message RaftAddServerResponse { +} + +message RaftRemoveServerRequest { + string id = 1; + bool force = 2; +} +message RaftRemoveServerResponse { +} + +message RaftListClusterServersRequest { +} +message RaftListClusterServersResponse { + message ClusterServers { + string id = 1; + string address = 2; + string suffrage = 3; // + } + repeated ClusterServers cluster_servers = 1; +} diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 427f3650b..4ffa7fb69 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -1,17 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: master.proto package master_pb import ( - context "context" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -25,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Heartbeat struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -53,6 +44,8 @@ type Heartbeat struct { DeletedEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,18,rep,name=deleted_ec_shards,json=deletedEcShards,proto3" json:"deleted_ec_shards,omitempty"` HasNoEcShards bool `protobuf:"varint,19,opt,name=has_no_ec_shards,json=hasNoEcShards,proto3" json:"has_no_ec_shards,omitempty"` MaxVolumeCounts map[string]uint32 `protobuf:"bytes,4,rep,name=max_volume_counts,json=maxVolumeCounts,proto3" json:"max_volume_counts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + GrpcPort uint32 `protobuf:"varint,20,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` + LocationUuids []string `protobuf:"bytes,21,rep,name=location_uuids,json=locationUuids,proto3" json:"location_uuids,omitempty"` } func (x *Heartbeat) Reset() { @@ -199,6 +192,20 @@ func (x *Heartbeat) GetMaxVolumeCounts() map[string]uint32 { return nil } +func (x *Heartbeat) GetGrpcPort() uint32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + +func (x *Heartbeat) GetLocationUuids() []string { + if x != nil { + return x.LocationUuids + } + return nil +} + type HeartbeatResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -209,6 +216,7 @@ type HeartbeatResponse struct { MetricsAddress string `protobuf:"bytes,3,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"` MetricsIntervalSeconds uint32 `protobuf:"varint,4,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"` StorageBackends []*StorageBackend `protobuf:"bytes,5,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"` + DuplicatedUuids []string `protobuf:"bytes,6,rep,name=duplicated_uuids,json=duplicatedUuids,proto3" json:"duplicated_uuids,omitempty"` } func (x *HeartbeatResponse) Reset() { @@ -278,6 +286,13 @@ func (x *HeartbeatResponse) GetStorageBackends() []*StorageBackend { return nil } +func (x *HeartbeatResponse) GetDuplicatedUuids() []string { + if x != nil { + return x.DuplicatedUuids + } + return nil +} + type VolumeInformationMessage struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -748,8 +763,10 @@ type KeepConnectedRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - GrpcPort uint32 `protobuf:"varint,2,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` + ClientType string `protobuf:"bytes,1,opt,name=client_type,json=clientType,proto3" json:"client_type,omitempty"` + ClientAddress string `protobuf:"bytes,3,opt,name=client_address,json=clientAddress,proto3" json:"client_address,omitempty"` + Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` + FilerGroup string `protobuf:"bytes,5,opt,name=filer_group,json=filerGroup,proto3" json:"filer_group,omitempty"` } func (x *KeepConnectedRequest) Reset() { @@ -784,18 +801,32 @@ func (*KeepConnectedRequest) Descriptor() ([]byte, []int) { return file_master_proto_rawDescGZIP(), []int{8} } -func (x *KeepConnectedRequest) GetName() string { +func (x *KeepConnectedRequest) GetClientType() string { if x != nil { - return x.Name + return x.ClientType } return "" } -func (x *KeepConnectedRequest) GetGrpcPort() uint32 { +func (x *KeepConnectedRequest) GetClientAddress() string { if x != nil { - return x.GrpcPort + return x.ClientAddress } - return 0 + return "" +} + +func (x *KeepConnectedRequest) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *KeepConnectedRequest) GetFilerGroup() string { + if x != nil { + return x.FilerGroup + } + return "" } type VolumeLocation struct { @@ -803,12 +834,15 @@ type VolumeLocation struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` - PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` - NewVids []uint32 `protobuf:"varint,3,rep,packed,name=new_vids,json=newVids,proto3" json:"new_vids,omitempty"` - DeletedVids []uint32 `protobuf:"varint,4,rep,packed,name=deleted_vids,json=deletedVids,proto3" json:"deleted_vids,omitempty"` - Leader string `protobuf:"bytes,5,opt,name=leader,proto3" json:"leader,omitempty"` // optional when leader is not itself - DataCenter string `protobuf:"bytes,6,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` // optional when DataCenter is in use + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` + NewVids []uint32 `protobuf:"varint,3,rep,packed,name=new_vids,json=newVids,proto3" json:"new_vids,omitempty"` + DeletedVids []uint32 `protobuf:"varint,4,rep,packed,name=deleted_vids,json=deletedVids,proto3" json:"deleted_vids,omitempty"` + Leader string `protobuf:"bytes,5,opt,name=leader,proto3" json:"leader,omitempty"` // optional when leader is not itself + DataCenter string `protobuf:"bytes,6,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` // optional when DataCenter is in use + GrpcPort uint32 `protobuf:"varint,7,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` + NewEcVids []uint32 `protobuf:"varint,8,rep,packed,name=new_ec_vids,json=newEcVids,proto3" json:"new_ec_vids,omitempty"` + DeletedEcVids []uint32 `protobuf:"varint,9,rep,packed,name=deleted_ec_vids,json=deletedEcVids,proto3" json:"deleted_ec_vids,omitempty"` } func (x *VolumeLocation) Reset() { @@ -885,6 +919,169 @@ func (x *VolumeLocation) GetDataCenter() string { return "" } +func (x *VolumeLocation) GetGrpcPort() uint32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + +func (x *VolumeLocation) GetNewEcVids() []uint32 { + if x != nil { + return x.NewEcVids + } + return nil +} + +func (x *VolumeLocation) GetDeletedEcVids() []uint32 { + if x != nil { + return x.DeletedEcVids + } + return nil +} + +type ClusterNodeUpdate struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeType string `protobuf:"bytes,1,opt,name=node_type,json=nodeType,proto3" json:"node_type,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + IsLeader bool `protobuf:"varint,3,opt,name=is_leader,json=isLeader,proto3" json:"is_leader,omitempty"` + IsAdd bool `protobuf:"varint,4,opt,name=is_add,json=isAdd,proto3" json:"is_add,omitempty"` + FilerGroup string `protobuf:"bytes,5,opt,name=filer_group,json=filerGroup,proto3" json:"filer_group,omitempty"` + CreatedAtNs int64 `protobuf:"varint,6,opt,name=created_at_ns,json=createdAtNs,proto3" json:"created_at_ns,omitempty"` +} + +func (x *ClusterNodeUpdate) Reset() { + *x = ClusterNodeUpdate{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClusterNodeUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClusterNodeUpdate) ProtoMessage() {} + +func (x *ClusterNodeUpdate) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClusterNodeUpdate.ProtoReflect.Descriptor instead. +func (*ClusterNodeUpdate) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{10} +} + +func (x *ClusterNodeUpdate) GetNodeType() string { + if x != nil { + return x.NodeType + } + return "" +} + +func (x *ClusterNodeUpdate) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *ClusterNodeUpdate) GetIsLeader() bool { + if x != nil { + return x.IsLeader + } + return false +} + +func (x *ClusterNodeUpdate) GetIsAdd() bool { + if x != nil { + return x.IsAdd + } + return false +} + +func (x *ClusterNodeUpdate) GetFilerGroup() string { + if x != nil { + return x.FilerGroup + } + return "" +} + +func (x *ClusterNodeUpdate) GetCreatedAtNs() int64 { + if x != nil { + return x.CreatedAtNs + } + return 0 +} + +type KeepConnectedResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + VolumeLocation *VolumeLocation `protobuf:"bytes,1,opt,name=volume_location,json=volumeLocation,proto3" json:"volume_location,omitempty"` + ClusterNodeUpdate *ClusterNodeUpdate `protobuf:"bytes,2,opt,name=cluster_node_update,json=clusterNodeUpdate,proto3" json:"cluster_node_update,omitempty"` +} + +func (x *KeepConnectedResponse) Reset() { + *x = KeepConnectedResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *KeepConnectedResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KeepConnectedResponse) ProtoMessage() {} + +func (x *KeepConnectedResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KeepConnectedResponse.ProtoReflect.Descriptor instead. +func (*KeepConnectedResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{11} +} + +func (x *KeepConnectedResponse) GetVolumeLocation() *VolumeLocation { + if x != nil { + return x.VolumeLocation + } + return nil +} + +func (x *KeepConnectedResponse) GetClusterNodeUpdate() *ClusterNodeUpdate { + if x != nil { + return x.ClusterNodeUpdate + } + return nil +} + type LookupVolumeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -897,7 +1094,7 @@ type LookupVolumeRequest struct { func (x *LookupVolumeRequest) Reset() { *x = LookupVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[10] + mi := &file_master_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -910,7 +1107,7 @@ func (x *LookupVolumeRequest) String() string { func (*LookupVolumeRequest) ProtoMessage() {} func (x *LookupVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[10] + mi := &file_master_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -923,7 +1120,7 @@ func (x *LookupVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupVolumeRequest.ProtoReflect.Descriptor instead. func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{10} + return file_master_proto_rawDescGZIP(), []int{12} } func (x *LookupVolumeRequest) GetVolumeOrFileIds() []string { @@ -951,7 +1148,7 @@ type LookupVolumeResponse struct { func (x *LookupVolumeResponse) Reset() { *x = LookupVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[11] + mi := &file_master_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -964,7 +1161,7 @@ func (x *LookupVolumeResponse) String() string { func (*LookupVolumeResponse) ProtoMessage() {} func (x *LookupVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[11] + mi := &file_master_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -977,7 +1174,7 @@ func (x *LookupVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupVolumeResponse.ProtoReflect.Descriptor instead. func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{11} + return file_master_proto_rawDescGZIP(), []int{13} } func (x *LookupVolumeResponse) GetVolumeIdLocations() []*LookupVolumeResponse_VolumeIdLocation { @@ -994,12 +1191,13 @@ type Location struct { Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` + GrpcPort uint32 `protobuf:"varint,3,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` } func (x *Location) Reset() { *x = Location{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[12] + mi := &file_master_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1012,7 +1210,7 @@ func (x *Location) String() string { func (*Location) ProtoMessage() {} func (x *Location) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[12] + mi := &file_master_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1025,7 +1223,7 @@ func (x *Location) ProtoReflect() protoreflect.Message { // Deprecated: Use Location.ProtoReflect.Descriptor instead. func (*Location) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{12} + return file_master_proto_rawDescGZIP(), []int{14} } func (x *Location) GetUrl() string { @@ -1042,6 +1240,13 @@ func (x *Location) GetPublicUrl() string { return "" } +func (x *Location) GetGrpcPort() uint32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + type AssignRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1062,7 +1267,7 @@ type AssignRequest struct { func (x *AssignRequest) Reset() { *x = AssignRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[13] + mi := &file_master_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1075,7 +1280,7 @@ func (x *AssignRequest) String() string { func (*AssignRequest) ProtoMessage() {} func (x *AssignRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[13] + mi := &file_master_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1088,7 +1293,7 @@ func (x *AssignRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AssignRequest.ProtoReflect.Descriptor instead. func (*AssignRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{13} + return file_master_proto_rawDescGZIP(), []int{15} } func (x *AssignRequest) GetCount() uint64 { @@ -1166,18 +1371,18 @@ type AssignResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Fid string `protobuf:"bytes,1,opt,name=fid,proto3" json:"fid,omitempty"` - Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` - PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` - Count uint64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` - Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"` - Auth string `protobuf:"bytes,6,opt,name=auth,proto3" json:"auth,omitempty"` + Fid string `protobuf:"bytes,1,opt,name=fid,proto3" json:"fid,omitempty"` + Count uint64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` + Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"` + Auth string `protobuf:"bytes,6,opt,name=auth,proto3" json:"auth,omitempty"` + Replicas []*Location `protobuf:"bytes,7,rep,name=replicas,proto3" json:"replicas,omitempty"` + Location *Location `protobuf:"bytes,8,opt,name=location,proto3" json:"location,omitempty"` } func (x *AssignResponse) Reset() { *x = AssignResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[14] + mi := &file_master_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1190,7 +1395,7 @@ func (x *AssignResponse) String() string { func (*AssignResponse) ProtoMessage() {} func (x *AssignResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[14] + mi := &file_master_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1203,7 +1408,7 @@ func (x *AssignResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AssignResponse.ProtoReflect.Descriptor instead. func (*AssignResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{14} + return file_master_proto_rawDescGZIP(), []int{16} } func (x *AssignResponse) GetFid() string { @@ -1213,20 +1418,6 @@ func (x *AssignResponse) GetFid() string { return "" } -func (x *AssignResponse) GetUrl() string { - if x != nil { - return x.Url - } - return "" -} - -func (x *AssignResponse) GetPublicUrl() string { - if x != nil { - return x.PublicUrl - } - return "" -} - func (x *AssignResponse) GetCount() uint64 { if x != nil { return x.Count @@ -1248,6 +1439,20 @@ func (x *AssignResponse) GetAuth() string { return "" } +func (x *AssignResponse) GetReplicas() []*Location { + if x != nil { + return x.Replicas + } + return nil +} + +func (x *AssignResponse) GetLocation() *Location { + if x != nil { + return x.Location + } + return nil +} + type StatisticsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1262,7 +1467,7 @@ type StatisticsRequest struct { func (x *StatisticsRequest) Reset() { *x = StatisticsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[15] + mi := &file_master_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1275,7 +1480,7 @@ func (x *StatisticsRequest) String() string { func (*StatisticsRequest) ProtoMessage() {} func (x *StatisticsRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[15] + mi := &file_master_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1288,7 +1493,7 @@ func (x *StatisticsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StatisticsRequest.ProtoReflect.Descriptor instead. func (*StatisticsRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{15} + return file_master_proto_rawDescGZIP(), []int{17} } func (x *StatisticsRequest) GetReplication() string { @@ -1332,7 +1537,7 @@ type StatisticsResponse struct { func (x *StatisticsResponse) Reset() { *x = StatisticsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[16] + mi := &file_master_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1345,7 +1550,7 @@ func (x *StatisticsResponse) String() string { func (*StatisticsResponse) ProtoMessage() {} func (x *StatisticsResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[16] + mi := &file_master_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1358,7 +1563,7 @@ func (x *StatisticsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StatisticsResponse.ProtoReflect.Descriptor instead. func (*StatisticsResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{16} + return file_master_proto_rawDescGZIP(), []int{18} } func (x *StatisticsResponse) GetTotalSize() uint64 { @@ -1396,7 +1601,7 @@ type Collection struct { func (x *Collection) Reset() { *x = Collection{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[17] + mi := &file_master_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1409,7 +1614,7 @@ func (x *Collection) String() string { func (*Collection) ProtoMessage() {} func (x *Collection) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[17] + mi := &file_master_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1422,7 +1627,7 @@ func (x *Collection) ProtoReflect() protoreflect.Message { // Deprecated: Use Collection.ProtoReflect.Descriptor instead. func (*Collection) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{17} + return file_master_proto_rawDescGZIP(), []int{19} } func (x *Collection) GetName() string { @@ -1444,7 +1649,7 @@ type CollectionListRequest struct { func (x *CollectionListRequest) Reset() { *x = CollectionListRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[18] + mi := &file_master_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1457,7 +1662,7 @@ func (x *CollectionListRequest) String() string { func (*CollectionListRequest) ProtoMessage() {} func (x *CollectionListRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[18] + mi := &file_master_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1470,7 +1675,7 @@ func (x *CollectionListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionListRequest.ProtoReflect.Descriptor instead. func (*CollectionListRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{18} + return file_master_proto_rawDescGZIP(), []int{20} } func (x *CollectionListRequest) GetIncludeNormalVolumes() bool { @@ -1498,7 +1703,7 @@ type CollectionListResponse struct { func (x *CollectionListResponse) Reset() { *x = CollectionListResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[19] + mi := &file_master_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1511,7 +1716,7 @@ func (x *CollectionListResponse) String() string { func (*CollectionListResponse) ProtoMessage() {} func (x *CollectionListResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[19] + mi := &file_master_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1524,7 +1729,7 @@ func (x *CollectionListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionListResponse.ProtoReflect.Descriptor instead. func (*CollectionListResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{19} + return file_master_proto_rawDescGZIP(), []int{21} } func (x *CollectionListResponse) GetCollections() []*Collection { @@ -1545,7 +1750,7 @@ type CollectionDeleteRequest struct { func (x *CollectionDeleteRequest) Reset() { *x = CollectionDeleteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[20] + mi := &file_master_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1558,7 +1763,7 @@ func (x *CollectionDeleteRequest) String() string { func (*CollectionDeleteRequest) ProtoMessage() {} func (x *CollectionDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[20] + mi := &file_master_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1571,7 +1776,7 @@ func (x *CollectionDeleteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionDeleteRequest.ProtoReflect.Descriptor instead. func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{20} + return file_master_proto_rawDescGZIP(), []int{22} } func (x *CollectionDeleteRequest) GetName() string { @@ -1590,7 +1795,7 @@ type CollectionDeleteResponse struct { func (x *CollectionDeleteResponse) Reset() { *x = CollectionDeleteResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[21] + mi := &file_master_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1603,7 +1808,7 @@ func (x *CollectionDeleteResponse) String() string { func (*CollectionDeleteResponse) ProtoMessage() {} func (x *CollectionDeleteResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[21] + mi := &file_master_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1616,7 +1821,7 @@ func (x *CollectionDeleteResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CollectionDeleteResponse.ProtoReflect.Descriptor instead. func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{21} + return file_master_proto_rawDescGZIP(), []int{23} } // @@ -1640,7 +1845,7 @@ type DiskInfo struct { func (x *DiskInfo) Reset() { *x = DiskInfo{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[22] + mi := &file_master_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1653,7 +1858,7 @@ func (x *DiskInfo) String() string { func (*DiskInfo) ProtoMessage() {} func (x *DiskInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[22] + mi := &file_master_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1666,7 +1871,7 @@ func (x *DiskInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DiskInfo.ProtoReflect.Descriptor instead. func (*DiskInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{22} + return file_master_proto_rawDescGZIP(), []int{24} } func (x *DiskInfo) GetType() string { @@ -1732,12 +1937,13 @@ type DataNodeInfo struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` DiskInfos map[string]*DiskInfo `protobuf:"bytes,2,rep,name=diskInfos,proto3" json:"diskInfos,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + GrpcPort uint32 `protobuf:"varint,3,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` } func (x *DataNodeInfo) Reset() { *x = DataNodeInfo{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[23] + mi := &file_master_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1750,7 +1956,7 @@ func (x *DataNodeInfo) String() string { func (*DataNodeInfo) ProtoMessage() {} func (x *DataNodeInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[23] + mi := &file_master_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1763,7 +1969,7 @@ func (x *DataNodeInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DataNodeInfo.ProtoReflect.Descriptor instead. func (*DataNodeInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{23} + return file_master_proto_rawDescGZIP(), []int{25} } func (x *DataNodeInfo) GetId() string { @@ -1780,6 +1986,13 @@ func (x *DataNodeInfo) GetDiskInfos() map[string]*DiskInfo { return nil } +func (x *DataNodeInfo) GetGrpcPort() uint32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + type RackInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1793,7 +2006,7 @@ type RackInfo struct { func (x *RackInfo) Reset() { *x = RackInfo{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[24] + mi := &file_master_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1806,7 +2019,7 @@ func (x *RackInfo) String() string { func (*RackInfo) ProtoMessage() {} func (x *RackInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[24] + mi := &file_master_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1819,7 +2032,7 @@ func (x *RackInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use RackInfo.ProtoReflect.Descriptor instead. func (*RackInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{24} + return file_master_proto_rawDescGZIP(), []int{26} } func (x *RackInfo) GetId() string { @@ -1856,7 +2069,7 @@ type DataCenterInfo struct { func (x *DataCenterInfo) Reset() { *x = DataCenterInfo{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[25] + mi := &file_master_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1869,7 +2082,7 @@ func (x *DataCenterInfo) String() string { func (*DataCenterInfo) ProtoMessage() {} func (x *DataCenterInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[25] + mi := &file_master_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1882,7 +2095,7 @@ func (x *DataCenterInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use DataCenterInfo.ProtoReflect.Descriptor instead. func (*DataCenterInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{25} + return file_master_proto_rawDescGZIP(), []int{27} } func (x *DataCenterInfo) GetId() string { @@ -1919,7 +2132,7 @@ type TopologyInfo struct { func (x *TopologyInfo) Reset() { *x = TopologyInfo{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[26] + mi := &file_master_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1932,7 +2145,7 @@ func (x *TopologyInfo) String() string { func (*TopologyInfo) ProtoMessage() {} func (x *TopologyInfo) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[26] + mi := &file_master_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1945,7 +2158,7 @@ func (x *TopologyInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use TopologyInfo.ProtoReflect.Descriptor instead. func (*TopologyInfo) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{26} + return file_master_proto_rawDescGZIP(), []int{28} } func (x *TopologyInfo) GetId() string { @@ -1978,7 +2191,7 @@ type VolumeListRequest struct { func (x *VolumeListRequest) Reset() { *x = VolumeListRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[27] + mi := &file_master_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1991,7 +2204,7 @@ func (x *VolumeListRequest) String() string { func (*VolumeListRequest) ProtoMessage() {} func (x *VolumeListRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[27] + mi := &file_master_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2004,7 +2217,7 @@ func (x *VolumeListRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeListRequest.ProtoReflect.Descriptor instead. func (*VolumeListRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{27} + return file_master_proto_rawDescGZIP(), []int{29} } type VolumeListResponse struct { @@ -2019,7 +2232,7 @@ type VolumeListResponse struct { func (x *VolumeListResponse) Reset() { *x = VolumeListResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[28] + mi := &file_master_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2032,7 +2245,7 @@ func (x *VolumeListResponse) String() string { func (*VolumeListResponse) ProtoMessage() {} func (x *VolumeListResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[28] + mi := &file_master_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2045,7 +2258,7 @@ func (x *VolumeListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeListResponse.ProtoReflect.Descriptor instead. func (*VolumeListResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{28} + return file_master_proto_rawDescGZIP(), []int{30} } func (x *VolumeListResponse) GetTopologyInfo() *TopologyInfo { @@ -2073,7 +2286,7 @@ type LookupEcVolumeRequest struct { func (x *LookupEcVolumeRequest) Reset() { *x = LookupEcVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[29] + mi := &file_master_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2086,7 +2299,7 @@ func (x *LookupEcVolumeRequest) String() string { func (*LookupEcVolumeRequest) ProtoMessage() {} func (x *LookupEcVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[29] + mi := &file_master_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2099,7 +2312,7 @@ func (x *LookupEcVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupEcVolumeRequest.ProtoReflect.Descriptor instead. func (*LookupEcVolumeRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{29} + return file_master_proto_rawDescGZIP(), []int{31} } func (x *LookupEcVolumeRequest) GetVolumeId() uint32 { @@ -2121,7 +2334,7 @@ type LookupEcVolumeResponse struct { func (x *LookupEcVolumeResponse) Reset() { *x = LookupEcVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[30] + mi := &file_master_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2134,7 +2347,7 @@ func (x *LookupEcVolumeResponse) String() string { func (*LookupEcVolumeResponse) ProtoMessage() {} func (x *LookupEcVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[30] + mi := &file_master_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2147,7 +2360,7 @@ func (x *LookupEcVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupEcVolumeResponse.ProtoReflect.Descriptor instead. func (*LookupEcVolumeResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{30} + return file_master_proto_rawDescGZIP(), []int{32} } func (x *LookupEcVolumeResponse) GetVolumeId() uint32 { @@ -2170,12 +2383,14 @@ type VacuumVolumeRequest struct { unknownFields protoimpl.UnknownFields GarbageThreshold float32 `protobuf:"fixed32,1,opt,name=garbage_threshold,json=garbageThreshold,proto3" json:"garbage_threshold,omitempty"` + VolumeId uint32 `protobuf:"varint,2,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + Collection string `protobuf:"bytes,3,opt,name=collection,proto3" json:"collection,omitempty"` } func (x *VacuumVolumeRequest) Reset() { *x = VacuumVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[31] + mi := &file_master_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2188,7 +2403,7 @@ func (x *VacuumVolumeRequest) String() string { func (*VacuumVolumeRequest) ProtoMessage() {} func (x *VacuumVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[31] + mi := &file_master_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2201,7 +2416,7 @@ func (x *VacuumVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VacuumVolumeRequest.ProtoReflect.Descriptor instead. func (*VacuumVolumeRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{31} + return file_master_proto_rawDescGZIP(), []int{33} } func (x *VacuumVolumeRequest) GetGarbageThreshold() float32 { @@ -2211,6 +2426,20 @@ func (x *VacuumVolumeRequest) GetGarbageThreshold() float32 { return 0 } +func (x *VacuumVolumeRequest) GetVolumeId() uint32 { + if x != nil { + return x.VolumeId + } + return 0 +} + +func (x *VacuumVolumeRequest) GetCollection() string { + if x != nil { + return x.Collection + } + return "" +} + type VacuumVolumeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2220,7 +2449,7 @@ type VacuumVolumeResponse struct { func (x *VacuumVolumeResponse) Reset() { *x = VacuumVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[32] + mi := &file_master_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2233,7 +2462,7 @@ func (x *VacuumVolumeResponse) String() string { func (*VacuumVolumeResponse) ProtoMessage() {} func (x *VacuumVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[32] + mi := &file_master_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2246,7 +2475,7 @@ func (x *VacuumVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VacuumVolumeResponse.ProtoReflect.Descriptor instead. func (*VacuumVolumeResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{32} + return file_master_proto_rawDescGZIP(), []int{34} } type GetMasterConfigurationRequest struct { @@ -2258,7 +2487,7 @@ type GetMasterConfigurationRequest struct { func (x *GetMasterConfigurationRequest) Reset() { *x = GetMasterConfigurationRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[33] + mi := &file_master_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2271,7 +2500,7 @@ func (x *GetMasterConfigurationRequest) String() string { func (*GetMasterConfigurationRequest) ProtoMessage() {} func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[33] + mi := &file_master_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2284,7 +2513,7 @@ func (x *GetMasterConfigurationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetMasterConfigurationRequest.ProtoReflect.Descriptor instead. func (*GetMasterConfigurationRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{33} + return file_master_proto_rawDescGZIP(), []int{35} } type GetMasterConfigurationResponse struct { @@ -2304,7 +2533,7 @@ type GetMasterConfigurationResponse struct { func (x *GetMasterConfigurationResponse) Reset() { *x = GetMasterConfigurationResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[34] + mi := &file_master_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2317,7 +2546,7 @@ func (x *GetMasterConfigurationResponse) String() string { func (*GetMasterConfigurationResponse) ProtoMessage() {} func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[34] + mi := &file_master_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2330,7 +2559,7 @@ func (x *GetMasterConfigurationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetMasterConfigurationResponse.ProtoReflect.Descriptor instead. func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{34} + return file_master_proto_rawDescGZIP(), []int{36} } func (x *GetMasterConfigurationResponse) GetMetricsAddress() string { @@ -2382,31 +2611,32 @@ func (x *GetMasterConfigurationResponse) GetVolumePreallocate() bool { return false } -type ListMasterClientsRequest struct { +type ListClusterNodesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields ClientType string `protobuf:"bytes,1,opt,name=client_type,json=clientType,proto3" json:"client_type,omitempty"` + FilerGroup string `protobuf:"bytes,2,opt,name=filer_group,json=filerGroup,proto3" json:"filer_group,omitempty"` } -func (x *ListMasterClientsRequest) Reset() { - *x = ListMasterClientsRequest{} +func (x *ListClusterNodesRequest) Reset() { + *x = ListClusterNodesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[35] + mi := &file_master_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListMasterClientsRequest) String() string { +func (x *ListClusterNodesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListMasterClientsRequest) ProtoMessage() {} +func (*ListClusterNodesRequest) ProtoMessage() {} -func (x *ListMasterClientsRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[35] +func (x *ListClusterNodesRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2417,43 +2647,50 @@ func (x *ListMasterClientsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListMasterClientsRequest.ProtoReflect.Descriptor instead. -func (*ListMasterClientsRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{35} +// Deprecated: Use ListClusterNodesRequest.ProtoReflect.Descriptor instead. +func (*ListClusterNodesRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{37} } -func (x *ListMasterClientsRequest) GetClientType() string { +func (x *ListClusterNodesRequest) GetClientType() string { if x != nil { return x.ClientType } return "" } -type ListMasterClientsResponse struct { +func (x *ListClusterNodesRequest) GetFilerGroup() string { + if x != nil { + return x.FilerGroup + } + return "" +} + +type ListClusterNodesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - GrpcAddresses []string `protobuf:"bytes,1,rep,name=grpc_addresses,json=grpcAddresses,proto3" json:"grpc_addresses,omitempty"` + ClusterNodes []*ListClusterNodesResponse_ClusterNode `protobuf:"bytes,1,rep,name=cluster_nodes,json=clusterNodes,proto3" json:"cluster_nodes,omitempty"` } -func (x *ListMasterClientsResponse) Reset() { - *x = ListMasterClientsResponse{} +func (x *ListClusterNodesResponse) Reset() { + *x = ListClusterNodesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[36] + mi := &file_master_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListMasterClientsResponse) String() string { +func (x *ListClusterNodesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListMasterClientsResponse) ProtoMessage() {} +func (*ListClusterNodesResponse) ProtoMessage() {} -func (x *ListMasterClientsResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[36] +func (x *ListClusterNodesResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2464,14 +2701,14 @@ func (x *ListMasterClientsResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListMasterClientsResponse.ProtoReflect.Descriptor instead. -func (*ListMasterClientsResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{36} +// Deprecated: Use ListClusterNodesResponse.ProtoReflect.Descriptor instead. +func (*ListClusterNodesResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{38} } -func (x *ListMasterClientsResponse) GetGrpcAddresses() []string { +func (x *ListClusterNodesResponse) GetClusterNodes() []*ListClusterNodesResponse_ClusterNode { if x != nil { - return x.GrpcAddresses + return x.ClusterNodes } return nil } @@ -2485,12 +2722,13 @@ type LeaseAdminTokenRequest struct { PreviousLockTime int64 `protobuf:"varint,2,opt,name=previous_lock_time,json=previousLockTime,proto3" json:"previous_lock_time,omitempty"` LockName string `protobuf:"bytes,3,opt,name=lock_name,json=lockName,proto3" json:"lock_name,omitempty"` ClientName string `protobuf:"bytes,4,opt,name=client_name,json=clientName,proto3" json:"client_name,omitempty"` + Message string `protobuf:"bytes,5,opt,name=message,proto3" json:"message,omitempty"` } func (x *LeaseAdminTokenRequest) Reset() { *x = LeaseAdminTokenRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[37] + mi := &file_master_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2503,7 +2741,7 @@ func (x *LeaseAdminTokenRequest) String() string { func (*LeaseAdminTokenRequest) ProtoMessage() {} func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[37] + mi := &file_master_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2516,7 +2754,7 @@ func (x *LeaseAdminTokenRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LeaseAdminTokenRequest.ProtoReflect.Descriptor instead. func (*LeaseAdminTokenRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{37} + return file_master_proto_rawDescGZIP(), []int{39} } func (x *LeaseAdminTokenRequest) GetPreviousToken() int64 { @@ -2547,6 +2785,13 @@ func (x *LeaseAdminTokenRequest) GetClientName() string { return "" } +func (x *LeaseAdminTokenRequest) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + type LeaseAdminTokenResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2559,7 +2804,7 @@ type LeaseAdminTokenResponse struct { func (x *LeaseAdminTokenResponse) Reset() { *x = LeaseAdminTokenResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[38] + mi := &file_master_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2572,7 +2817,7 @@ func (x *LeaseAdminTokenResponse) String() string { func (*LeaseAdminTokenResponse) ProtoMessage() {} func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[38] + mi := &file_master_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2585,7 +2830,7 @@ func (x *LeaseAdminTokenResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LeaseAdminTokenResponse.ProtoReflect.Descriptor instead. func (*LeaseAdminTokenResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{38} + return file_master_proto_rawDescGZIP(), []int{40} } func (x *LeaseAdminTokenResponse) GetToken() int64 { @@ -2615,7 +2860,7 @@ type ReleaseAdminTokenRequest struct { func (x *ReleaseAdminTokenRequest) Reset() { *x = ReleaseAdminTokenRequest{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[39] + mi := &file_master_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2628,7 +2873,7 @@ func (x *ReleaseAdminTokenRequest) String() string { func (*ReleaseAdminTokenRequest) ProtoMessage() {} func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[39] + mi := &file_master_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2641,7 +2886,7 @@ func (x *ReleaseAdminTokenRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseAdminTokenRequest.ProtoReflect.Descriptor instead. func (*ReleaseAdminTokenRequest) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{39} + return file_master_proto_rawDescGZIP(), []int{41} } func (x *ReleaseAdminTokenRequest) GetPreviousToken() int64 { @@ -2674,7 +2919,7 @@ type ReleaseAdminTokenResponse struct { func (x *ReleaseAdminTokenResponse) Reset() { *x = ReleaseAdminTokenResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[40] + mi := &file_master_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2687,7 +2932,7 @@ func (x *ReleaseAdminTokenResponse) String() string { func (*ReleaseAdminTokenResponse) ProtoMessage() {} func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[40] + mi := &file_master_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2700,21 +2945,20 @@ func (x *ReleaseAdminTokenResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseAdminTokenResponse.ProtoReflect.Descriptor instead. func (*ReleaseAdminTokenResponse) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{40} + return file_master_proto_rawDescGZIP(), []int{42} } -type SuperBlockExtra_ErasureCoding struct { +type PingRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Data uint32 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` - Parity uint32 `protobuf:"varint,2,opt,name=parity,proto3" json:"parity,omitempty"` - VolumeIds []uint32 `protobuf:"varint,3,rep,packed,name=volume_ids,json=volumeIds,proto3" json:"volume_ids,omitempty"` + Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // default to ping itself + TargetType string `protobuf:"bytes,2,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` } -func (x *SuperBlockExtra_ErasureCoding) Reset() { - *x = SuperBlockExtra_ErasureCoding{} +func (x *PingRequest) Reset() { + *x = PingRequest{} if protoimpl.UnsafeEnabled { mi := &file_master_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2722,13 +2966,13 @@ func (x *SuperBlockExtra_ErasureCoding) Reset() { } } -func (x *SuperBlockExtra_ErasureCoding) String() string { +func (x *PingRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} +func (*PingRequest) ProtoMessage() {} -func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { +func (x *PingRequest) ProtoReflect() protoreflect.Message { mi := &file_master_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2740,45 +2984,37 @@ func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SuperBlockExtra_ErasureCoding.ProtoReflect.Descriptor instead. -func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{7, 0} -} - -func (x *SuperBlockExtra_ErasureCoding) GetData() uint32 { - if x != nil { - return x.Data - } - return 0 +// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. +func (*PingRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{43} } -func (x *SuperBlockExtra_ErasureCoding) GetParity() uint32 { +func (x *PingRequest) GetTarget() string { if x != nil { - return x.Parity + return x.Target } - return 0 + return "" } -func (x *SuperBlockExtra_ErasureCoding) GetVolumeIds() []uint32 { +func (x *PingRequest) GetTargetType() string { if x != nil { - return x.VolumeIds + return x.TargetType } - return nil + return "" } -type LookupVolumeResponse_VolumeIdLocation struct { +type PingResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - VolumeOrFileId string `protobuf:"bytes,1,opt,name=volume_or_file_id,json=volumeOrFileId,proto3" json:"volume_or_file_id,omitempty"` - Locations []*Location `protobuf:"bytes,2,rep,name=locations,proto3" json:"locations,omitempty"` - Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` - Auth string `protobuf:"bytes,4,opt,name=auth,proto3" json:"auth,omitempty"` + StartTimeNs int64 `protobuf:"varint,1,opt,name=start_time_ns,json=startTimeNs,proto3" json:"start_time_ns,omitempty"` + RemoteTimeNs int64 `protobuf:"varint,2,opt,name=remote_time_ns,json=remoteTimeNs,proto3" json:"remote_time_ns,omitempty"` + StopTimeNs int64 `protobuf:"varint,3,opt,name=stop_time_ns,json=stopTimeNs,proto3" json:"stop_time_ns,omitempty"` } -func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { - *x = LookupVolumeResponse_VolumeIdLocation{} +func (x *PingResponse) Reset() { + *x = PingResponse{} if protoimpl.UnsafeEnabled { mi := &file_master_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2786,13 +3022,13 @@ func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { } } -func (x *LookupVolumeResponse_VolumeIdLocation) String() string { +func (x *PingResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} +func (*PingResponse) ProtoMessage() {} -func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message { +func (x *PingResponse) ProtoReflect() protoreflect.Message { mi := &file_master_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -2804,65 +3040,118 @@ func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Mess return mi.MessageOf(x) } -// Deprecated: Use LookupVolumeResponse_VolumeIdLocation.ProtoReflect.Descriptor instead. -func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{11, 0} +// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. +func (*PingResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{44} } -func (x *LookupVolumeResponse_VolumeIdLocation) GetVolumeOrFileId() string { +func (x *PingResponse) GetStartTimeNs() int64 { if x != nil { - return x.VolumeOrFileId + return x.StartTimeNs } - return "" + return 0 } -func (x *LookupVolumeResponse_VolumeIdLocation) GetLocations() []*Location { +func (x *PingResponse) GetRemoteTimeNs() int64 { if x != nil { - return x.Locations + return x.RemoteTimeNs } - return nil + return 0 } -func (x *LookupVolumeResponse_VolumeIdLocation) GetError() string { +func (x *PingResponse) GetStopTimeNs() int64 { if x != nil { - return x.Error + return x.StopTimeNs + } + return 0 +} + +type RaftAddServerRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + Voter bool `protobuf:"varint,3,opt,name=voter,proto3" json:"voter,omitempty"` +} + +func (x *RaftAddServerRequest) Reset() { + *x = RaftAddServerRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[45] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RaftAddServerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftAddServerRequest) ProtoMessage() {} + +func (x *RaftAddServerRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[45] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftAddServerRequest.ProtoReflect.Descriptor instead. +func (*RaftAddServerRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{45} +} + +func (x *RaftAddServerRequest) GetId() string { + if x != nil { + return x.Id } return "" } -func (x *LookupVolumeResponse_VolumeIdLocation) GetAuth() string { +func (x *RaftAddServerRequest) GetAddress() string { if x != nil { - return x.Auth + return x.Address } return "" } -type LookupEcVolumeResponse_EcShardIdLocation struct { +func (x *RaftAddServerRequest) GetVoter() bool { + if x != nil { + return x.Voter + } + return false +} + +type RaftAddServerResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` - Locations []*Location `protobuf:"bytes,2,rep,name=locations,proto3" json:"locations,omitempty"` } -func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() { - *x = LookupEcVolumeResponse_EcShardIdLocation{} +func (x *RaftAddServerResponse) Reset() { + *x = RaftAddServerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_master_proto_msgTypes[49] + mi := &file_master_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string { +func (x *RaftAddServerResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {} +func (*RaftAddServerResponse) ProtoMessage() {} -func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[49] +func (x *RaftAddServerResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2873,31 +3162,518 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.M return mi.MessageOf(x) } -// Deprecated: Use LookupEcVolumeResponse_EcShardIdLocation.ProtoReflect.Descriptor instead. -func (*LookupEcVolumeResponse_EcShardIdLocation) Descriptor() ([]byte, []int) { - return file_master_proto_rawDescGZIP(), []int{30, 0} +// Deprecated: Use RaftAddServerResponse.ProtoReflect.Descriptor instead. +func (*RaftAddServerResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{46} } -func (x *LookupEcVolumeResponse_EcShardIdLocation) GetShardId() uint32 { - if x != nil { - return x.ShardId - } - return 0 +type RaftRemoveServerRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"` } -func (x *LookupEcVolumeResponse_EcShardIdLocation) GetLocations() []*Location { - if x != nil { - return x.Locations +func (x *RaftRemoveServerRequest) Reset() { + *x = RaftRemoveServerRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -var File_master_proto protoreflect.FileDescriptor +func (x *RaftRemoveServerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} -var file_master_proto_rawDesc = []byte{ - 0x0a, 0x0c, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x22, 0xfc, 0x06, 0x0a, 0x09, 0x48, 0x65, - 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, +func (*RaftRemoveServerRequest) ProtoMessage() {} + +func (x *RaftRemoveServerRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[47] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftRemoveServerRequest.ProtoReflect.Descriptor instead. +func (*RaftRemoveServerRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{47} +} + +func (x *RaftRemoveServerRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *RaftRemoveServerRequest) GetForce() bool { + if x != nil { + return x.Force + } + return false +} + +type RaftRemoveServerResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RaftRemoveServerResponse) Reset() { + *x = RaftRemoveServerResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RaftRemoveServerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftRemoveServerResponse) ProtoMessage() {} + +func (x *RaftRemoveServerResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[48] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftRemoveServerResponse.ProtoReflect.Descriptor instead. +func (*RaftRemoveServerResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{48} +} + +type RaftListClusterServersRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RaftListClusterServersRequest) Reset() { + *x = RaftListClusterServersRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RaftListClusterServersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftListClusterServersRequest) ProtoMessage() {} + +func (x *RaftListClusterServersRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[49] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftListClusterServersRequest.ProtoReflect.Descriptor instead. +func (*RaftListClusterServersRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{49} +} + +type RaftListClusterServersResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ClusterServers []*RaftListClusterServersResponse_ClusterServers `protobuf:"bytes,1,rep,name=cluster_servers,json=clusterServers,proto3" json:"cluster_servers,omitempty"` +} + +func (x *RaftListClusterServersResponse) Reset() { + *x = RaftListClusterServersResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RaftListClusterServersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftListClusterServersResponse) ProtoMessage() {} + +func (x *RaftListClusterServersResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[50] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftListClusterServersResponse.ProtoReflect.Descriptor instead. +func (*RaftListClusterServersResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{50} +} + +func (x *RaftListClusterServersResponse) GetClusterServers() []*RaftListClusterServersResponse_ClusterServers { + if x != nil { + return x.ClusterServers + } + return nil +} + +type SuperBlockExtra_ErasureCoding struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data uint32 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` + Parity uint32 `protobuf:"varint,2,opt,name=parity,proto3" json:"parity,omitempty"` + VolumeIds []uint32 `protobuf:"varint,3,rep,packed,name=volume_ids,json=volumeIds,proto3" json:"volume_ids,omitempty"` +} + +func (x *SuperBlockExtra_ErasureCoding) Reset() { + *x = SuperBlockExtra_ErasureCoding{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SuperBlockExtra_ErasureCoding) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} + +func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[53] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuperBlockExtra_ErasureCoding.ProtoReflect.Descriptor instead. +func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{7, 0} +} + +func (x *SuperBlockExtra_ErasureCoding) GetData() uint32 { + if x != nil { + return x.Data + } + return 0 +} + +func (x *SuperBlockExtra_ErasureCoding) GetParity() uint32 { + if x != nil { + return x.Parity + } + return 0 +} + +func (x *SuperBlockExtra_ErasureCoding) GetVolumeIds() []uint32 { + if x != nil { + return x.VolumeIds + } + return nil +} + +type LookupVolumeResponse_VolumeIdLocation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + VolumeOrFileId string `protobuf:"bytes,1,opt,name=volume_or_file_id,json=volumeOrFileId,proto3" json:"volume_or_file_id,omitempty"` + Locations []*Location `protobuf:"bytes,2,rep,name=locations,proto3" json:"locations,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + Auth string `protobuf:"bytes,4,opt,name=auth,proto3" json:"auth,omitempty"` +} + +func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { + *x = LookupVolumeResponse_VolumeIdLocation{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LookupVolumeResponse_VolumeIdLocation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} + +func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[54] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupVolumeResponse_VolumeIdLocation.ProtoReflect.Descriptor instead. +func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{13, 0} +} + +func (x *LookupVolumeResponse_VolumeIdLocation) GetVolumeOrFileId() string { + if x != nil { + return x.VolumeOrFileId + } + return "" +} + +func (x *LookupVolumeResponse_VolumeIdLocation) GetLocations() []*Location { + if x != nil { + return x.Locations + } + return nil +} + +func (x *LookupVolumeResponse_VolumeIdLocation) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *LookupVolumeResponse_VolumeIdLocation) GetAuth() string { + if x != nil { + return x.Auth + } + return "" +} + +type LookupEcVolumeResponse_EcShardIdLocation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + Locations []*Location `protobuf:"bytes,2,rep,name=locations,proto3" json:"locations,omitempty"` +} + +func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() { + *x = LookupEcVolumeResponse_EcShardIdLocation{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[59] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {} + +func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[59] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupEcVolumeResponse_EcShardIdLocation.ProtoReflect.Descriptor instead. +func (*LookupEcVolumeResponse_EcShardIdLocation) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{32, 0} +} + +func (x *LookupEcVolumeResponse_EcShardIdLocation) GetShardId() uint32 { + if x != nil { + return x.ShardId + } + return 0 +} + +func (x *LookupEcVolumeResponse_EcShardIdLocation) GetLocations() []*Location { + if x != nil { + return x.Locations + } + return nil +} + +type ListClusterNodesResponse_ClusterNode struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + IsLeader bool `protobuf:"varint,3,opt,name=is_leader,json=isLeader,proto3" json:"is_leader,omitempty"` + CreatedAtNs int64 `protobuf:"varint,4,opt,name=created_at_ns,json=createdAtNs,proto3" json:"created_at_ns,omitempty"` +} + +func (x *ListClusterNodesResponse_ClusterNode) Reset() { + *x = ListClusterNodesResponse_ClusterNode{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[60] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListClusterNodesResponse_ClusterNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListClusterNodesResponse_ClusterNode) ProtoMessage() {} + +func (x *ListClusterNodesResponse_ClusterNode) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[60] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListClusterNodesResponse_ClusterNode.ProtoReflect.Descriptor instead. +func (*ListClusterNodesResponse_ClusterNode) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{38, 0} +} + +func (x *ListClusterNodesResponse_ClusterNode) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *ListClusterNodesResponse_ClusterNode) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *ListClusterNodesResponse_ClusterNode) GetIsLeader() bool { + if x != nil { + return x.IsLeader + } + return false +} + +func (x *ListClusterNodesResponse_ClusterNode) GetCreatedAtNs() int64 { + if x != nil { + return x.CreatedAtNs + } + return 0 +} + +type RaftListClusterServersResponse_ClusterServers struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"` // +} + +func (x *RaftListClusterServersResponse_ClusterServers) Reset() { + *x = RaftListClusterServersResponse_ClusterServers{} + if protoimpl.UnsafeEnabled { + mi := &file_master_proto_msgTypes[61] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RaftListClusterServersResponse_ClusterServers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftListClusterServersResponse_ClusterServers) ProtoMessage() {} + +func (x *RaftListClusterServersResponse_ClusterServers) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[61] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftListClusterServersResponse_ClusterServers.ProtoReflect.Descriptor instead. +func (*RaftListClusterServersResponse_ClusterServers) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{50, 0} +} + +func (x *RaftListClusterServersResponse_ClusterServers) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *RaftListClusterServersResponse_ClusterServers) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *RaftListClusterServersResponse_ClusterServers) GetSuffrage() string { + if x != nil { + return x.Suffrage + } + return "" +} + +var File_master_proto protoreflect.FileDescriptor + +var file_master_proto_rawDesc = []byte{ + 0x0a, 0x0c, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x22, 0xc0, 0x07, 0x0a, 0x09, 0x48, 0x65, + 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -2948,476 +3724,606 @@ var file_master_proto_rawDesc = []byte{ 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x2e, 0x4d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x80, 0x02, 0x0a, 0x11, 0x48, 0x65, 0x61, - 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, - 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x16, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x22, 0x98, 0x04, 0x0a, 0x18, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, - 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x64, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x0b, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, - 0x0a, 0x12, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x64, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x5f, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, 0x6c, 0x61, - 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x74, - 0x74, 0x6c, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x72, 0x65, - 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, 0x6f, - 0x6d, 0x70, 0x61, 0x63, 0x74, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, - 0x12, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x73, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, - 0x69, 0x65, 0x64, 0x41, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, - 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, - 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc5, 0x01, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, + 0x75, 0x6e, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x75, 0x75, + 0x69, 0x64, 0x73, 0x18, 0x15, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x55, 0x75, 0x69, 0x64, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x02, 0x0a, + 0x11, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x38, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, + 0x29, 0x0a, 0x10, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x75, + 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x64, 0x75, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x55, 0x75, 0x69, 0x64, 0x73, 0x22, 0x98, 0x04, 0x0a, 0x18, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x63, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0b, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x0a, + 0x12, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, + 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, + 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x5f, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x74, 0x74, - 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0f, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x92, - 0x01, 0x0a, 0x1f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x63, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, - 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x65, 0x63, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x42, 0x69, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, - 0x79, 0x70, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, - 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x49, 0x0a, 0x0a, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, - 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0xbe, 0x01, - 0x0a, 0x0f, 0x53, 0x75, 0x70, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x45, 0x78, 0x74, 0x72, - 0x61, 0x12, 0x4f, 0x0a, 0x0e, 0x65, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x64, - 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x70, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, - 0x45, 0x78, 0x74, 0x72, 0x61, 0x2e, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, - 0x69, 0x6e, 0x67, 0x52, 0x0d, 0x65, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, 0x69, - 0x6e, 0x67, 0x1a, 0x5a, 0x0a, 0x0d, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, - 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x69, 0x74, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79, 0x12, - 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0d, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, 0x47, + 0x6c, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x72, 0x65, 0x76, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, 0x6f, 0x6d, + 0x70, 0x61, 0x63, 0x74, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, + 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, + 0x65, 0x64, 0x41, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, + 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc5, 0x01, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x53, 0x68, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x5f, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x10, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, 0x6c, 0x61, 0x63, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, + 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x74, 0x74, 0x6c, + 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x92, 0x01, + 0x0a, 0x1f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x63, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x62, 0x69, + 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x65, 0x63, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x42, 0x69, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, + 0x70, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, + 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x49, 0x0a, 0x0a, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0xbe, 0x01, 0x0a, + 0x0f, 0x53, 0x75, 0x70, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x45, 0x78, 0x74, 0x72, 0x61, + 0x12, 0x4f, 0x0a, 0x0e, 0x65, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x69, + 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x70, 0x65, 0x72, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x45, + 0x78, 0x74, 0x72, 0x61, 0x2e, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, 0x69, + 0x6e, 0x67, 0x52, 0x0d, 0x65, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, 0x69, 0x6e, + 0x67, 0x1a, 0x5a, 0x0a, 0x0d, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x64, 0x69, + 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1d, + 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, - 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, - 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xb8, 0x01, 0x0a, 0x0e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, - 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x6e, - 0x65, 0x77, 0x5f, 0x76, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x6e, - 0x65, 0x77, 0x56, 0x69, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x5f, 0x76, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0b, 0x64, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x64, 0x56, 0x69, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, - 0x65, 0x72, 0x22, 0x62, 0x0a, 0x13, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x12, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x5f, 0x6f, 0x72, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4f, 0x72, 0x46, - 0x69, 0x6c, 0x65, 0x49, 0x64, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x95, 0x02, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, - 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x60, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x1a, 0x9a, 0x01, 0x0a, 0x10, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x4c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x6f, 0x72, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4f, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x49, - 0x64, 0x12, 0x31, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, - 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x22, 0x3b, - 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, - 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x22, 0xd0, 0x02, 0x0a, 0x0d, - 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, - 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, - 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, - 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x16, 0x6d, 0x65, 0x6d, - 0x6f, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, - 0x5f, 0x6d, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x4d, 0x61, 0x70, 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x12, 0x32, 0x0a, - 0x15, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x13, 0x57, 0x72, - 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x93, - 0x01, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x66, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x66, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x9d, 0x02, 0x0a, 0x0e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, + 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x12, 0x19, 0x0a, + 0x08, 0x6e, 0x65, 0x77, 0x5f, 0x76, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, + 0x07, 0x6e, 0x65, 0x77, 0x56, 0x69, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x5f, 0x76, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0b, + 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x56, 0x69, 0x64, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, + 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, + 0x6e, 0x74, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x1e, 0x0a, 0x0b, 0x6e, 0x65, 0x77, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x69, 0x64, 0x73, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x65, 0x77, 0x45, 0x63, 0x56, 0x69, 0x64, + 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x63, 0x5f, + 0x76, 0x69, 0x64, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x64, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x45, 0x63, 0x56, 0x69, 0x64, 0x73, 0x22, 0xc3, 0x01, 0x0a, 0x11, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x41, 0x64, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x22, 0x0a, 0x0d, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x4e, 0x73, 0x22, + 0xa9, 0x01, 0x0a, 0x15, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0f, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4c, 0x0a, + 0x13, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, + 0x64, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x11, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x62, 0x0a, 0x13, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x12, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x6f, 0x72, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4f, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x73, 0x12, + 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x95, 0x02, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, + 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x9a, 0x01, 0x0a, 0x10, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x29, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x6f, 0x72, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x4f, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x22, 0x58, 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, - 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x55, 0x72, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x61, 0x75, 0x74, 0x68, 0x22, 0x84, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, - 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, - 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, - 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x12, 0x53, - 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65, - 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, - 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0a, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x7b, - 0x0a, 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, 0x2c, 0x0a, - 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x22, 0x51, 0x0a, 0x16, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2d, - 0x0a, 0x17, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1a, 0x0a, - 0x18, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x91, 0x03, 0x0a, 0x08, 0x44, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0b, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x28, 0x0a, - 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x66, 0x72, 0x65, 0x65, 0x5f, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0f, 0x66, 0x72, 0x65, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x11, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x46, 0x0a, 0x0c, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x6e, - 0x66, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0b, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x50, 0x0a, 0x0e, 0x65, - 0x63, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x07, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, - 0x0c, 0x65, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x2e, 0x0a, - 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb7, 0x01, - 0x0a, 0x0c, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x44, - 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, - 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, - 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, - 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, 0x0e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf0, 0x01, 0x0a, 0x08, 0x52, 0x61, 0x63, 0x6b, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x3f, 0x0a, 0x0f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, - 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x40, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, - 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x64, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, 0x0e, 0x44, 0x69, 0x73, 0x6b, 0x49, - 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xef, 0x01, 0x0a, 0x0e, 0x44, - 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, - 0x0a, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, - 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x72, 0x61, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x12, 0x46, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x2e, - 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, - 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, 0x0e, 0x44, 0x69, 0x73, - 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfe, 0x01, 0x0a, - 0x0c, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x45, 0x0a, - 0x11, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x66, - 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x44, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, - 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, 0x0e, 0x44, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x13, 0x0a, - 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x83, 0x01, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x0d, 0x74, 0x6f, 0x70, - 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, - 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0c, 0x74, 0x6f, 0x70, 0x6f, 0x6c, - 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2f, 0x0a, 0x14, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x6d, 0x62, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, 0x7a, - 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x4d, 0x62, 0x22, 0x34, 0x0a, 0x15, 0x4c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0xfb, - 0x01, 0x0a, 0x16, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, - 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x4c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, - 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x61, 0x0a, 0x11, 0x45, 0x63, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x19, - 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x09, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, + 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x55, 0x72, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, + 0x74, 0x22, 0xd0, 0x02, 0x0a, 0x0d, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, + 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1f, 0x0a, + 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, + 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, + 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, + 0x32, 0x0a, 0x16, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x5f, 0x6d, 0x61, + 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4d, 0x61, 0x70, 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, + 0x65, 0x4d, 0x62, 0x12, 0x32, 0x0a, 0x15, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x13, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, + 0x54, 0x79, 0x70, 0x65, 0x22, 0xc4, 0x01, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x66, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, + 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x08, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x12, 0x2f, 0x0a, 0x08, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x42, 0x0a, 0x13, - 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x67, 0x61, 0x72, 0x62, 0x61, 0x67, 0x65, 0x5f, 0x74, - 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x10, - 0x67, 0x61, 0x72, 0x62, 0x61, 0x67, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, - 0x22, 0x16, 0x0a, 0x14, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x4d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf3, 0x02, 0x0a, 0x1e, 0x47, 0x65, - 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x0f, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, - 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, - 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, - 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, - 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x12, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x30, - 0x0a, 0x15, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x5f, 0x6d, 0x5f, 0x62, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x4d, 0x42, - 0x12, 0x2d, 0x0a, 0x12, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x61, 0x6c, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x50, 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x22, - 0x3b, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, + 0x6e, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x01, 0x0a, 0x11, + 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, + 0x70, 0x65, 0x22, 0x6f, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, + 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x7b, 0x0a, 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, + 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, + 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, + 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x65, 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x73, 0x22, 0x51, 0x0a, 0x16, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2d, 0x0a, 0x17, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1a, 0x0a, 0x18, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x91, 0x03, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, + 0x6d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2a, + 0x0a, 0x11, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x66, 0x72, 0x65, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x46, 0x0a, 0x0c, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0b, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, + 0x6f, 0x73, 0x12, 0x50, 0x0a, 0x0e, 0x65, 0x63, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, + 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0c, 0x65, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0c, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x44, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, + 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, + 0x6f, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x67, + 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, + 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x1a, 0x51, 0x0a, 0x0e, 0x44, 0x69, 0x73, 0x6b, + 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf0, 0x01, 0x0a, 0x08, + 0x52, 0x61, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3f, 0x0a, 0x0f, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, + 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x40, 0x0a, 0x09, 0x64, 0x69, 0x73, + 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x63, 0x6b, 0x49, 0x6e, 0x66, + 0x6f, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, 0x0e, 0x44, + 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xef, + 0x01, 0x0a, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x52, 0x61, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x72, 0x61, 0x63, 0x6b, + 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x46, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, + 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, 0x0a, + 0x0e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, + 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0xfe, 0x01, 0x0a, 0x0c, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x45, 0x0a, 0x11, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, + 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x44, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, + 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x61, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, + 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x51, + 0x0a, 0x0e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, + 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x13, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x83, 0x01, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, + 0x0d, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0c, 0x74, + 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2f, 0x0a, 0x14, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x5f, 0x6d, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x4d, 0x62, 0x22, 0x34, 0x0a, 0x15, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x49, 0x64, 0x22, 0xfb, 0x01, 0x0a, 0x16, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x61, 0x0a, + 0x11, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x31, 0x0a, + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x7f, 0x0a, 0x13, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x67, 0x61, 0x72, 0x62, 0x61, + 0x67, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x02, 0x52, 0x10, 0x67, 0x61, 0x72, 0x62, 0x61, 0x67, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x16, 0x0a, 0x14, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, + 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf3, 0x02, 0x0a, 0x1e, 0x47, + 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, + 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, + 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, + 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, + 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, + 0x30, 0x0a, 0x15, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x6d, 0x5f, 0x62, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x4d, + 0x42, 0x12, 0x2d, 0x0a, 0x12, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x61, + 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x50, 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, + 0x22, 0x5b, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, + 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x42, 0x0a, 0x19, - 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, - 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, - 0x22, 0xab, 0x01, 0x0a, 0x16, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, - 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, - 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, - 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, - 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4d, - 0x0a, 0x17, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x1c, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x54, 0x73, 0x4e, 0x73, 0x22, 0x8c, 0x01, - 0x0a, 0x18, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, - 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, - 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, - 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x1b, 0x0a, 0x19, - 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xca, 0x09, 0x0a, 0x07, 0x53, 0x65, + 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xf5, 0x01, + 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, + 0x64, 0x65, 0x52, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, + 0x1a, 0x82, 0x01, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, + 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x41, 0x74, 0x4e, 0x73, 0x22, 0xc5, 0x01, 0x0a, 0x16, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, + 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, + 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65, 0x76, 0x69, + 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63, + 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x4d, 0x0a, + 0x17, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1c, + 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x54, 0x73, 0x4e, 0x73, 0x22, 0x8c, 0x01, 0x0a, + 0x18, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, + 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x5f, 0x6c, 0x6f, 0x63, + 0x6b, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x70, 0x72, + 0x65, 0x76, 0x69, 0x6f, 0x75, 0x73, 0x4c, 0x6f, 0x63, 0x6b, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, + 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x1b, 0x0a, 0x19, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x46, 0x0a, 0x0b, 0x50, 0x69, 0x6e, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, + 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x22, 0x7a, 0x0a, 0x0c, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x73, 0x74, + 0x6f, 0x70, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x22, 0x56, 0x0a, 0x14, + 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x6f, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, + 0x6f, 0x74, 0x65, 0x72, 0x22, 0x17, 0x0a, 0x15, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3f, 0x0a, + 0x17, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x22, 0x1a, + 0x0a, 0x18, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x52, 0x61, + 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xdb, 0x01, 0x0a, 0x1e, + 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, + 0x0a, 0x0f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x52, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x1a, 0x56, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, + 0x08, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x32, 0xaf, 0x0c, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, - 0x12, 0x51, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, + 0x12, 0x58, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x00, 0x28, - 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, - 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, - 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, - 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, - 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, + 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, + 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, + 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, + 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, + 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, + 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, + 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, + 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, - 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x2e, 0x6d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x73, - 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, - 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, - 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, - 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, - 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, - 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, - 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, - 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, + 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, + 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, + 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50, 0x69, + 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, + 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, + 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, + 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, + 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, + 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, + 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, + 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3432,7 +4338,7 @@ func file_master_proto_rawDescGZIP() []byte { return file_master_proto_rawDescData } -var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 50) +var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 62) var file_master_proto_goTypes = []interface{}{ (*Heartbeat)(nil), // 0: master_pb.Heartbeat (*HeartbeatResponse)(nil), // 1: master_pb.HeartbeatResponse @@ -3444,46 +4350,58 @@ var file_master_proto_goTypes = []interface{}{ (*SuperBlockExtra)(nil), // 7: master_pb.SuperBlockExtra (*KeepConnectedRequest)(nil), // 8: master_pb.KeepConnectedRequest (*VolumeLocation)(nil), // 9: master_pb.VolumeLocation - (*LookupVolumeRequest)(nil), // 10: master_pb.LookupVolumeRequest - (*LookupVolumeResponse)(nil), // 11: master_pb.LookupVolumeResponse - (*Location)(nil), // 12: master_pb.Location - (*AssignRequest)(nil), // 13: master_pb.AssignRequest - (*AssignResponse)(nil), // 14: master_pb.AssignResponse - (*StatisticsRequest)(nil), // 15: master_pb.StatisticsRequest - (*StatisticsResponse)(nil), // 16: master_pb.StatisticsResponse - (*Collection)(nil), // 17: master_pb.Collection - (*CollectionListRequest)(nil), // 18: master_pb.CollectionListRequest - (*CollectionListResponse)(nil), // 19: master_pb.CollectionListResponse - (*CollectionDeleteRequest)(nil), // 20: master_pb.CollectionDeleteRequest - (*CollectionDeleteResponse)(nil), // 21: master_pb.CollectionDeleteResponse - (*DiskInfo)(nil), // 22: master_pb.DiskInfo - (*DataNodeInfo)(nil), // 23: master_pb.DataNodeInfo - (*RackInfo)(nil), // 24: master_pb.RackInfo - (*DataCenterInfo)(nil), // 25: master_pb.DataCenterInfo - (*TopologyInfo)(nil), // 26: master_pb.TopologyInfo - (*VolumeListRequest)(nil), // 27: master_pb.VolumeListRequest - (*VolumeListResponse)(nil), // 28: master_pb.VolumeListResponse - (*LookupEcVolumeRequest)(nil), // 29: master_pb.LookupEcVolumeRequest - (*LookupEcVolumeResponse)(nil), // 30: master_pb.LookupEcVolumeResponse - (*VacuumVolumeRequest)(nil), // 31: master_pb.VacuumVolumeRequest - (*VacuumVolumeResponse)(nil), // 32: master_pb.VacuumVolumeResponse - (*GetMasterConfigurationRequest)(nil), // 33: master_pb.GetMasterConfigurationRequest - (*GetMasterConfigurationResponse)(nil), // 34: master_pb.GetMasterConfigurationResponse - (*ListMasterClientsRequest)(nil), // 35: master_pb.ListMasterClientsRequest - (*ListMasterClientsResponse)(nil), // 36: master_pb.ListMasterClientsResponse - (*LeaseAdminTokenRequest)(nil), // 37: master_pb.LeaseAdminTokenRequest - (*LeaseAdminTokenResponse)(nil), // 38: master_pb.LeaseAdminTokenResponse - (*ReleaseAdminTokenRequest)(nil), // 39: master_pb.ReleaseAdminTokenRequest - (*ReleaseAdminTokenResponse)(nil), // 40: master_pb.ReleaseAdminTokenResponse - nil, // 41: master_pb.Heartbeat.MaxVolumeCountsEntry - nil, // 42: master_pb.StorageBackend.PropertiesEntry - (*SuperBlockExtra_ErasureCoding)(nil), // 43: master_pb.SuperBlockExtra.ErasureCoding - (*LookupVolumeResponse_VolumeIdLocation)(nil), // 44: master_pb.LookupVolumeResponse.VolumeIdLocation - nil, // 45: master_pb.DataNodeInfo.DiskInfosEntry - nil, // 46: master_pb.RackInfo.DiskInfosEntry - nil, // 47: master_pb.DataCenterInfo.DiskInfosEntry - nil, // 48: master_pb.TopologyInfo.DiskInfosEntry - (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 49: master_pb.LookupEcVolumeResponse.EcShardIdLocation + (*ClusterNodeUpdate)(nil), // 10: master_pb.ClusterNodeUpdate + (*KeepConnectedResponse)(nil), // 11: master_pb.KeepConnectedResponse + (*LookupVolumeRequest)(nil), // 12: master_pb.LookupVolumeRequest + (*LookupVolumeResponse)(nil), // 13: master_pb.LookupVolumeResponse + (*Location)(nil), // 14: master_pb.Location + (*AssignRequest)(nil), // 15: master_pb.AssignRequest + (*AssignResponse)(nil), // 16: master_pb.AssignResponse + (*StatisticsRequest)(nil), // 17: master_pb.StatisticsRequest + (*StatisticsResponse)(nil), // 18: master_pb.StatisticsResponse + (*Collection)(nil), // 19: master_pb.Collection + (*CollectionListRequest)(nil), // 20: master_pb.CollectionListRequest + (*CollectionListResponse)(nil), // 21: master_pb.CollectionListResponse + (*CollectionDeleteRequest)(nil), // 22: master_pb.CollectionDeleteRequest + (*CollectionDeleteResponse)(nil), // 23: master_pb.CollectionDeleteResponse + (*DiskInfo)(nil), // 24: master_pb.DiskInfo + (*DataNodeInfo)(nil), // 25: master_pb.DataNodeInfo + (*RackInfo)(nil), // 26: master_pb.RackInfo + (*DataCenterInfo)(nil), // 27: master_pb.DataCenterInfo + (*TopologyInfo)(nil), // 28: master_pb.TopologyInfo + (*VolumeListRequest)(nil), // 29: master_pb.VolumeListRequest + (*VolumeListResponse)(nil), // 30: master_pb.VolumeListResponse + (*LookupEcVolumeRequest)(nil), // 31: master_pb.LookupEcVolumeRequest + (*LookupEcVolumeResponse)(nil), // 32: master_pb.LookupEcVolumeResponse + (*VacuumVolumeRequest)(nil), // 33: master_pb.VacuumVolumeRequest + (*VacuumVolumeResponse)(nil), // 34: master_pb.VacuumVolumeResponse + (*GetMasterConfigurationRequest)(nil), // 35: master_pb.GetMasterConfigurationRequest + (*GetMasterConfigurationResponse)(nil), // 36: master_pb.GetMasterConfigurationResponse + (*ListClusterNodesRequest)(nil), // 37: master_pb.ListClusterNodesRequest + (*ListClusterNodesResponse)(nil), // 38: master_pb.ListClusterNodesResponse + (*LeaseAdminTokenRequest)(nil), // 39: master_pb.LeaseAdminTokenRequest + (*LeaseAdminTokenResponse)(nil), // 40: master_pb.LeaseAdminTokenResponse + (*ReleaseAdminTokenRequest)(nil), // 41: master_pb.ReleaseAdminTokenRequest + (*ReleaseAdminTokenResponse)(nil), // 42: master_pb.ReleaseAdminTokenResponse + (*PingRequest)(nil), // 43: master_pb.PingRequest + (*PingResponse)(nil), // 44: master_pb.PingResponse + (*RaftAddServerRequest)(nil), // 45: master_pb.RaftAddServerRequest + (*RaftAddServerResponse)(nil), // 46: master_pb.RaftAddServerResponse + (*RaftRemoveServerRequest)(nil), // 47: master_pb.RaftRemoveServerRequest + (*RaftRemoveServerResponse)(nil), // 48: master_pb.RaftRemoveServerResponse + (*RaftListClusterServersRequest)(nil), // 49: master_pb.RaftListClusterServersRequest + (*RaftListClusterServersResponse)(nil), // 50: master_pb.RaftListClusterServersResponse + nil, // 51: master_pb.Heartbeat.MaxVolumeCountsEntry + nil, // 52: master_pb.StorageBackend.PropertiesEntry + (*SuperBlockExtra_ErasureCoding)(nil), // 53: master_pb.SuperBlockExtra.ErasureCoding + (*LookupVolumeResponse_VolumeIdLocation)(nil), // 54: master_pb.LookupVolumeResponse.VolumeIdLocation + nil, // 55: master_pb.DataNodeInfo.DiskInfosEntry + nil, // 56: master_pb.RackInfo.DiskInfosEntry + nil, // 57: master_pb.DataCenterInfo.DiskInfosEntry + nil, // 58: master_pb.TopologyInfo.DiskInfosEntry + (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 59: master_pb.LookupEcVolumeResponse.EcShardIdLocation + (*ListClusterNodesResponse_ClusterNode)(nil), // 60: master_pb.ListClusterNodesResponse.ClusterNode + (*RaftListClusterServersResponse_ClusterServers)(nil), // 61: master_pb.RaftListClusterServersResponse.ClusterServers } var file_master_proto_depIdxs = []int32{ 2, // 0: master_pb.Heartbeat.volumes:type_name -> master_pb.VolumeInformationMessage @@ -3492,63 +4410,77 @@ var file_master_proto_depIdxs = []int32{ 4, // 3: master_pb.Heartbeat.ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 4: master_pb.Heartbeat.new_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 5: master_pb.Heartbeat.deleted_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage - 41, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry + 51, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry 5, // 7: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend - 42, // 8: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry - 43, // 9: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding - 44, // 10: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation - 17, // 11: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection - 2, // 12: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage - 4, // 13: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage - 45, // 14: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry - 23, // 15: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo - 46, // 16: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry - 24, // 17: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo - 47, // 18: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry - 25, // 19: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo - 48, // 20: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry - 26, // 21: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo - 49, // 22: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation - 5, // 23: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend - 12, // 24: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location - 22, // 25: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 22, // 26: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 22, // 27: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 22, // 28: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 12, // 29: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location - 0, // 30: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat - 8, // 31: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest - 10, // 32: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest - 13, // 33: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest - 15, // 34: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest - 18, // 35: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest - 20, // 36: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest - 27, // 37: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest - 29, // 38: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest - 31, // 39: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest - 33, // 40: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest - 35, // 41: master_pb.Seaweed.ListMasterClients:input_type -> master_pb.ListMasterClientsRequest - 37, // 42: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest - 39, // 43: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest - 1, // 44: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse - 9, // 45: master_pb.Seaweed.KeepConnected:output_type -> master_pb.VolumeLocation - 11, // 46: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse - 14, // 47: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse - 16, // 48: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse - 19, // 49: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse - 21, // 50: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse - 28, // 51: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse - 30, // 52: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse - 32, // 53: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse - 34, // 54: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse - 36, // 55: master_pb.Seaweed.ListMasterClients:output_type -> master_pb.ListMasterClientsResponse - 38, // 56: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse - 40, // 57: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse - 44, // [44:58] is the sub-list for method output_type - 30, // [30:44] is the sub-list for method input_type - 30, // [30:30] is the sub-list for extension type_name - 30, // [30:30] is the sub-list for extension extendee - 0, // [0:30] is the sub-list for field type_name + 52, // 8: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry + 53, // 9: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding + 9, // 10: master_pb.KeepConnectedResponse.volume_location:type_name -> master_pb.VolumeLocation + 10, // 11: master_pb.KeepConnectedResponse.cluster_node_update:type_name -> master_pb.ClusterNodeUpdate + 54, // 12: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation + 14, // 13: master_pb.AssignResponse.replicas:type_name -> master_pb.Location + 14, // 14: master_pb.AssignResponse.location:type_name -> master_pb.Location + 19, // 15: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection + 2, // 16: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage + 4, // 17: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage + 55, // 18: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry + 25, // 19: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo + 56, // 20: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry + 26, // 21: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo + 57, // 22: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry + 27, // 23: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo + 58, // 24: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry + 28, // 25: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo + 59, // 26: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation + 5, // 27: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend + 60, // 28: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode + 61, // 29: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers + 14, // 30: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location + 24, // 31: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 24, // 32: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 24, // 33: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 24, // 34: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 14, // 35: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location + 0, // 36: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat + 8, // 37: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest + 12, // 38: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest + 15, // 39: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest + 17, // 40: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest + 20, // 41: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest + 22, // 42: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest + 29, // 43: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest + 31, // 44: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest + 33, // 45: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest + 35, // 46: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest + 37, // 47: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest + 39, // 48: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest + 41, // 49: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest + 43, // 50: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest + 49, // 51: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest + 45, // 52: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest + 47, // 53: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest + 1, // 54: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse + 11, // 55: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse + 13, // 56: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse + 16, // 57: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse + 18, // 58: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse + 21, // 59: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse + 23, // 60: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse + 30, // 61: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse + 32, // 62: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse + 34, // 63: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse + 36, // 64: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse + 38, // 65: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse + 40, // 66: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse + 42, // 67: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse + 44, // 68: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse + 50, // 69: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse + 46, // 70: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse + 48, // 71: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse + 54, // [54:72] is the sub-list for method output_type + 36, // [36:54] is the sub-list for method input_type + 36, // [36:36] is the sub-list for extension type_name + 36, // [36:36] is the sub-list for extension extendee + 0, // [0:36] is the sub-list for field type_name } func init() { file_master_proto_init() } @@ -3678,7 +4610,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupVolumeRequest); i { + switch v := v.(*ClusterNodeUpdate); i { case 0: return &v.state case 1: @@ -3690,7 +4622,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupVolumeResponse); i { + switch v := v.(*KeepConnectedResponse); i { case 0: return &v.state case 1: @@ -3702,7 +4634,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Location); i { + switch v := v.(*LookupVolumeRequest); i { case 0: return &v.state case 1: @@ -3714,7 +4646,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AssignRequest); i { + switch v := v.(*LookupVolumeResponse); i { case 0: return &v.state case 1: @@ -3726,7 +4658,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AssignResponse); i { + switch v := v.(*Location); i { case 0: return &v.state case 1: @@ -3738,7 +4670,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StatisticsRequest); i { + switch v := v.(*AssignRequest); i { case 0: return &v.state case 1: @@ -3750,7 +4682,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StatisticsResponse); i { + switch v := v.(*AssignResponse); i { case 0: return &v.state case 1: @@ -3762,7 +4694,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Collection); i { + switch v := v.(*StatisticsRequest); i { case 0: return &v.state case 1: @@ -3774,7 +4706,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionListRequest); i { + switch v := v.(*StatisticsResponse); i { case 0: return &v.state case 1: @@ -3786,7 +4718,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionListResponse); i { + switch v := v.(*Collection); i { case 0: return &v.state case 1: @@ -3798,7 +4730,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionDeleteRequest); i { + switch v := v.(*CollectionListRequest); i { case 0: return &v.state case 1: @@ -3810,7 +4742,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CollectionDeleteResponse); i { + switch v := v.(*CollectionListResponse); i { case 0: return &v.state case 1: @@ -3822,7 +4754,7 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DiskInfo); i { + switch v := v.(*CollectionDeleteRequest); i { case 0: return &v.state case 1: @@ -3834,6 +4766,30 @@ func file_master_proto_init() { } } file_master_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CollectionDeleteResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DiskInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataNodeInfo); i { case 0: return &v.state @@ -3845,8 +4801,128 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RackInfo); i { + file_master_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RackInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataCenterInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TopologyInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VolumeListRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VolumeListResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LookupEcVolumeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LookupEcVolumeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VacuumVolumeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VacuumVolumeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetMasterConfigurationRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_master_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetMasterConfigurationResponse); i { case 0: return &v.state case 1: @@ -3857,8 +4933,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DataCenterInfo); i { + file_master_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListClusterNodesRequest); i { case 0: return &v.state case 1: @@ -3869,8 +4945,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TopologyInfo); i { + file_master_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListClusterNodesResponse); i { case 0: return &v.state case 1: @@ -3881,8 +4957,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeListRequest); i { + file_master_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LeaseAdminTokenRequest); i { case 0: return &v.state case 1: @@ -3893,8 +4969,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeListResponse); i { + file_master_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LeaseAdminTokenResponse); i { case 0: return &v.state case 1: @@ -3905,8 +4981,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupEcVolumeRequest); i { + file_master_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReleaseAdminTokenRequest); i { case 0: return &v.state case 1: @@ -3917,8 +4993,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupEcVolumeResponse); i { + file_master_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReleaseAdminTokenResponse); i { case 0: return &v.state case 1: @@ -3929,8 +5005,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VacuumVolumeRequest); i { + file_master_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PingRequest); i { case 0: return &v.state case 1: @@ -3941,8 +5017,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VacuumVolumeResponse); i { + file_master_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PingResponse); i { case 0: return &v.state case 1: @@ -3953,8 +5029,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetMasterConfigurationRequest); i { + file_master_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftAddServerRequest); i { case 0: return &v.state case 1: @@ -3965,8 +5041,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetMasterConfigurationResponse); i { + file_master_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftAddServerResponse); i { case 0: return &v.state case 1: @@ -3977,8 +5053,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListMasterClientsRequest); i { + file_master_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftRemoveServerRequest); i { case 0: return &v.state case 1: @@ -3989,8 +5065,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListMasterClientsResponse); i { + file_master_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftRemoveServerResponse); i { case 0: return &v.state case 1: @@ -4001,8 +5077,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LeaseAdminTokenRequest); i { + file_master_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftListClusterServersRequest); i { case 0: return &v.state case 1: @@ -4013,8 +5089,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LeaseAdminTokenResponse); i { + file_master_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftListClusterServersResponse); i { case 0: return &v.state case 1: @@ -4025,8 +5101,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReleaseAdminTokenRequest); i { + file_master_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SuperBlockExtra_ErasureCoding); i { case 0: return &v.state case 1: @@ -4037,8 +5113,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReleaseAdminTokenResponse); i { + file_master_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LookupVolumeResponse_VolumeIdLocation); i { case 0: return &v.state case 1: @@ -4049,8 +5125,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SuperBlockExtra_ErasureCoding); i { + file_master_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LookupEcVolumeResponse_EcShardIdLocation); i { case 0: return &v.state case 1: @@ -4061,8 +5137,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupVolumeResponse_VolumeIdLocation); i { + file_master_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListClusterNodesResponse_ClusterNode); i { case 0: return &v.state case 1: @@ -4073,8 +5149,8 @@ func file_master_proto_init() { return nil } } - file_master_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LookupEcVolumeResponse_EcShardIdLocation); i { + file_master_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RaftListClusterServersResponse_ClusterServers); i { case 0: return &v.state case 1: @@ -4092,7 +5168,7 @@ func file_master_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_master_proto_rawDesc, NumEnums: 0, - NumMessages: 50, + NumMessages: 62, NumExtensions: 0, NumServices: 1, }, @@ -4105,616 +5181,3 @@ func file_master_proto_init() { file_master_proto_goTypes = nil file_master_proto_depIdxs = nil } - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// SeaweedClient is the client API for Seaweed service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SeaweedClient interface { - SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) - KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) - LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) - Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error) - Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) - CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) - CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) - VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error) - LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error) - VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error) - GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) - ListMasterClients(ctx context.Context, in *ListMasterClientsRequest, opts ...grpc.CallOption) (*ListMasterClientsResponse, error) - LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error) - ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error) -} - -type seaweedClient struct { - cc grpc.ClientConnInterface -} - -func NewSeaweedClient(cc grpc.ClientConnInterface) SeaweedClient { - return &seaweedClient{cc} -} - -func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) { - stream, err := c.cc.NewStream(ctx, &_Seaweed_serviceDesc.Streams[0], "/master_pb.Seaweed/SendHeartbeat", opts...) - if err != nil { - return nil, err - } - x := &seaweedSendHeartbeatClient{stream} - return x, nil -} - -type Seaweed_SendHeartbeatClient interface { - Send(*Heartbeat) error - Recv() (*HeartbeatResponse, error) - grpc.ClientStream -} - -type seaweedSendHeartbeatClient struct { - grpc.ClientStream -} - -func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error { - return x.ClientStream.SendMsg(m) -} - -func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) { - m := new(HeartbeatResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) { - stream, err := c.cc.NewStream(ctx, &_Seaweed_serviceDesc.Streams[1], "/master_pb.Seaweed/KeepConnected", opts...) - if err != nil { - return nil, err - } - x := &seaweedKeepConnectedClient{stream} - return x, nil -} - -type Seaweed_KeepConnectedClient interface { - Send(*KeepConnectedRequest) error - Recv() (*VolumeLocation, error) - grpc.ClientStream -} - -type seaweedKeepConnectedClient struct { - grpc.ClientStream -} - -func (x *seaweedKeepConnectedClient) Send(m *KeepConnectedRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *seaweedKeepConnectedClient) Recv() (*VolumeLocation, error) { - m := new(VolumeLocation) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedClient) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) { - out := new(LookupVolumeResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LookupVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error) { - out := new(AssignResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Assign", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { - out := new(StatisticsResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Statistics", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) { - out := new(CollectionListResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/CollectionList", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) { - out := new(CollectionDeleteResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/CollectionDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error) { - out := new(VolumeListResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/VolumeList", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error) { - out := new(LookupEcVolumeResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LookupEcVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error) { - out := new(VacuumVolumeResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/VacuumVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) { - out := new(GetMasterConfigurationResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/GetMasterConfiguration", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) ListMasterClients(ctx context.Context, in *ListMasterClientsRequest, opts ...grpc.CallOption) (*ListMasterClientsResponse, error) { - out := new(ListMasterClientsResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/ListMasterClients", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error) { - out := new(LeaseAdminTokenResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LeaseAdminToken", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedClient) ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error) { - out := new(ReleaseAdminTokenResponse) - err := c.cc.Invoke(ctx, "/master_pb.Seaweed/ReleaseAdminToken", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SeaweedServer is the server API for Seaweed service. -type SeaweedServer interface { - SendHeartbeat(Seaweed_SendHeartbeatServer) error - KeepConnected(Seaweed_KeepConnectedServer) error - LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) - Assign(context.Context, *AssignRequest) (*AssignResponse, error) - Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) - CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) - CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) - VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error) - LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) - VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) - GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) - ListMasterClients(context.Context, *ListMasterClientsRequest) (*ListMasterClientsResponse, error) - LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error) - ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) -} - -// UnimplementedSeaweedServer can be embedded to have forward compatible implementations. -type UnimplementedSeaweedServer struct { -} - -func (*UnimplementedSeaweedServer) SendHeartbeat(Seaweed_SendHeartbeatServer) error { - return status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") -} -func (*UnimplementedSeaweedServer) KeepConnected(Seaweed_KeepConnectedServer) error { - return status.Errorf(codes.Unimplemented, "method KeepConnected not implemented") -} -func (*UnimplementedSeaweedServer) LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupVolume not implemented") -} -func (*UnimplementedSeaweedServer) Assign(context.Context, *AssignRequest) (*AssignResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Assign not implemented") -} -func (*UnimplementedSeaweedServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") -} -func (*UnimplementedSeaweedServer) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CollectionList not implemented") -} -func (*UnimplementedSeaweedServer) CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CollectionDelete not implemented") -} -func (*UnimplementedSeaweedServer) VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeList not implemented") -} -func (*UnimplementedSeaweedServer) LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupEcVolume not implemented") -} -func (*UnimplementedSeaweedServer) VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolume not implemented") -} -func (*UnimplementedSeaweedServer) GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetMasterConfiguration not implemented") -} -func (*UnimplementedSeaweedServer) ListMasterClients(context.Context, *ListMasterClientsRequest) (*ListMasterClientsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ListMasterClients not implemented") -} -func (*UnimplementedSeaweedServer) LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LeaseAdminToken not implemented") -} -func (*UnimplementedSeaweedServer) ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReleaseAdminToken not implemented") -} - -func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) { - s.RegisterService(&_Seaweed_serviceDesc, srv) -} - -func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream}) -} - -type Seaweed_SendHeartbeatServer interface { - Send(*HeartbeatResponse) error - Recv() (*Heartbeat, error) - grpc.ServerStream -} - -type seaweedSendHeartbeatServer struct { - grpc.ServerStream -} - -func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) { - m := new(Heartbeat) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _Seaweed_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SeaweedServer).KeepConnected(&seaweedKeepConnectedServer{stream}) -} - -type Seaweed_KeepConnectedServer interface { - Send(*VolumeLocation) error - Recv() (*KeepConnectedRequest, error) - grpc.ServerStream -} - -type seaweedKeepConnectedServer struct { - grpc.ServerStream -} - -func (x *seaweedKeepConnectedServer) Send(m *VolumeLocation) error { - return x.ServerStream.SendMsg(m) -} - -func (x *seaweedKeepConnectedServer) Recv() (*KeepConnectedRequest, error) { - m := new(KeepConnectedRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _Seaweed_LookupVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LookupVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).LookupVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/LookupVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).LookupVolume(ctx, req.(*LookupVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_Assign_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AssignRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).Assign(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/Assign", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).Assign(ctx, req.(*AssignRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StatisticsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).Statistics(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/Statistics", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).Statistics(ctx, req.(*StatisticsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_CollectionList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CollectionListRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).CollectionList(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/CollectionList", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).CollectionList(ctx, req.(*CollectionListRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_CollectionDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CollectionDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).CollectionDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/CollectionDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).CollectionDelete(ctx, req.(*CollectionDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_VolumeList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeListRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).VolumeList(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/VolumeList", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).VolumeList(ctx, req.(*VolumeListRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_LookupEcVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LookupEcVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).LookupEcVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/LookupEcVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).LookupEcVolume(ctx, req.(*LookupEcVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_VacuumVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VacuumVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).VacuumVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/VacuumVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).VacuumVolume(ctx, req.(*VacuumVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_GetMasterConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetMasterConfigurationRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).GetMasterConfiguration(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/GetMasterConfiguration", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).GetMasterConfiguration(ctx, req.(*GetMasterConfigurationRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_ListMasterClients_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ListMasterClientsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).ListMasterClients(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/ListMasterClients", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).ListMasterClients(ctx, req.(*ListMasterClientsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_LeaseAdminToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(LeaseAdminTokenRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).LeaseAdminToken(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/LeaseAdminToken", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).LeaseAdminToken(ctx, req.(*LeaseAdminTokenRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Seaweed_ReleaseAdminToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReleaseAdminTokenRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedServer).ReleaseAdminToken(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/master_pb.Seaweed/ReleaseAdminToken", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedServer).ReleaseAdminToken(ctx, req.(*ReleaseAdminTokenRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _Seaweed_serviceDesc = grpc.ServiceDesc{ - ServiceName: "master_pb.Seaweed", - HandlerType: (*SeaweedServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "LookupVolume", - Handler: _Seaweed_LookupVolume_Handler, - }, - { - MethodName: "Assign", - Handler: _Seaweed_Assign_Handler, - }, - { - MethodName: "Statistics", - Handler: _Seaweed_Statistics_Handler, - }, - { - MethodName: "CollectionList", - Handler: _Seaweed_CollectionList_Handler, - }, - { - MethodName: "CollectionDelete", - Handler: _Seaweed_CollectionDelete_Handler, - }, - { - MethodName: "VolumeList", - Handler: _Seaweed_VolumeList_Handler, - }, - { - MethodName: "LookupEcVolume", - Handler: _Seaweed_LookupEcVolume_Handler, - }, - { - MethodName: "VacuumVolume", - Handler: _Seaweed_VacuumVolume_Handler, - }, - { - MethodName: "GetMasterConfiguration", - Handler: _Seaweed_GetMasterConfiguration_Handler, - }, - { - MethodName: "ListMasterClients", - Handler: _Seaweed_ListMasterClients_Handler, - }, - { - MethodName: "LeaseAdminToken", - Handler: _Seaweed_LeaseAdminToken_Handler, - }, - { - MethodName: "ReleaseAdminToken", - Handler: _Seaweed_ReleaseAdminToken_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "SendHeartbeat", - Handler: _Seaweed_SendHeartbeat_Handler, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "KeepConnected", - Handler: _Seaweed_KeepConnected_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "master.proto", -} diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go new file mode 100644 index 000000000..f8b11e8c5 --- /dev/null +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -0,0 +1,778 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package master_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedClient is the client API for Seaweed service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedClient interface { + SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) + KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) + LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) + Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error) + Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) + CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) + CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) + VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error) + LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error) + VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error) + GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) + ListClusterNodes(ctx context.Context, in *ListClusterNodesRequest, opts ...grpc.CallOption) (*ListClusterNodesResponse, error) + LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error) + ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error) + Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) + RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error) + RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error) + RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) +} + +type seaweedClient struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedClient(cc grpc.ClientConnInterface) SeaweedClient { + return &seaweedClient{cc} +} + +func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) { + stream, err := c.cc.NewStream(ctx, &Seaweed_ServiceDesc.Streams[0], "/master_pb.Seaweed/SendHeartbeat", opts...) + if err != nil { + return nil, err + } + x := &seaweedSendHeartbeatClient{stream} + return x, nil +} + +type Seaweed_SendHeartbeatClient interface { + Send(*Heartbeat) error + Recv() (*HeartbeatResponse, error) + grpc.ClientStream +} + +type seaweedSendHeartbeatClient struct { + grpc.ClientStream +} + +func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) { + m := new(HeartbeatResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) { + stream, err := c.cc.NewStream(ctx, &Seaweed_ServiceDesc.Streams[1], "/master_pb.Seaweed/KeepConnected", opts...) + if err != nil { + return nil, err + } + x := &seaweedKeepConnectedClient{stream} + return x, nil +} + +type Seaweed_KeepConnectedClient interface { + Send(*KeepConnectedRequest) error + Recv() (*KeepConnectedResponse, error) + grpc.ClientStream +} + +type seaweedKeepConnectedClient struct { + grpc.ClientStream +} + +func (x *seaweedKeepConnectedClient) Send(m *KeepConnectedRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedKeepConnectedClient) Recv() (*KeepConnectedResponse, error) { + m := new(KeepConnectedResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedClient) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) { + out := new(LookupVolumeResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LookupVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error) { + out := new(AssignResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Assign", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { + out := new(StatisticsResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Statistics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) { + out := new(CollectionListResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/CollectionList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) { + out := new(CollectionDeleteResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/CollectionDelete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error) { + out := new(VolumeListResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/VolumeList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error) { + out := new(LookupEcVolumeResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LookupEcVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) VacuumVolume(ctx context.Context, in *VacuumVolumeRequest, opts ...grpc.CallOption) (*VacuumVolumeResponse, error) { + out := new(VacuumVolumeResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/VacuumVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) { + out := new(GetMasterConfigurationResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/GetMasterConfiguration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) ListClusterNodes(ctx context.Context, in *ListClusterNodesRequest, opts ...grpc.CallOption) (*ListClusterNodesResponse, error) { + out := new(ListClusterNodesResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/ListClusterNodes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error) { + out := new(LeaseAdminTokenResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/LeaseAdminToken", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error) { + out := new(ReleaseAdminTokenResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/ReleaseAdminToken", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { + out := new(PingResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error) { + out := new(RaftListClusterServersResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftListClusterServers", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error) { + out := new(RaftAddServerResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftAddServer", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) { + out := new(RaftRemoveServerResponse) + err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftRemoveServer", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SeaweedServer is the server API for Seaweed service. +// All implementations must embed UnimplementedSeaweedServer +// for forward compatibility +type SeaweedServer interface { + SendHeartbeat(Seaweed_SendHeartbeatServer) error + KeepConnected(Seaweed_KeepConnectedServer) error + LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) + Assign(context.Context, *AssignRequest) (*AssignResponse, error) + Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) + CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) + CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) + VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error) + LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) + VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) + GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) + ListClusterNodes(context.Context, *ListClusterNodesRequest) (*ListClusterNodesResponse, error) + LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error) + ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) + Ping(context.Context, *PingRequest) (*PingResponse, error) + RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error) + RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error) + RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) + mustEmbedUnimplementedSeaweedServer() +} + +// UnimplementedSeaweedServer must be embedded to have forward compatible implementations. +type UnimplementedSeaweedServer struct { +} + +func (UnimplementedSeaweedServer) SendHeartbeat(Seaweed_SendHeartbeatServer) error { + return status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") +} +func (UnimplementedSeaweedServer) KeepConnected(Seaweed_KeepConnectedServer) error { + return status.Errorf(codes.Unimplemented, "method KeepConnected not implemented") +} +func (UnimplementedSeaweedServer) LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LookupVolume not implemented") +} +func (UnimplementedSeaweedServer) Assign(context.Context, *AssignRequest) (*AssignResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Assign not implemented") +} +func (UnimplementedSeaweedServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") +} +func (UnimplementedSeaweedServer) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CollectionList not implemented") +} +func (UnimplementedSeaweedServer) CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CollectionDelete not implemented") +} +func (UnimplementedSeaweedServer) VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeList not implemented") +} +func (UnimplementedSeaweedServer) LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LookupEcVolume not implemented") +} +func (UnimplementedSeaweedServer) VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VacuumVolume not implemented") +} +func (UnimplementedSeaweedServer) GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMasterConfiguration not implemented") +} +func (UnimplementedSeaweedServer) ListClusterNodes(context.Context, *ListClusterNodesRequest) (*ListClusterNodesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListClusterNodes not implemented") +} +func (UnimplementedSeaweedServer) LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method LeaseAdminToken not implemented") +} +func (UnimplementedSeaweedServer) ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseAdminToken not implemented") +} +func (UnimplementedSeaweedServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedSeaweedServer) RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RaftListClusterServers not implemented") +} +func (UnimplementedSeaweedServer) RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RaftAddServer not implemented") +} +func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RaftRemoveServer not implemented") +} +func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {} + +// UnsafeSeaweedServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedServer will +// result in compilation errors. +type UnsafeSeaweedServer interface { + mustEmbedUnimplementedSeaweedServer() +} + +func RegisterSeaweedServer(s grpc.ServiceRegistrar, srv SeaweedServer) { + s.RegisterService(&Seaweed_ServiceDesc, srv) +} + +func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream}) +} + +type Seaweed_SendHeartbeatServer interface { + Send(*HeartbeatResponse) error + Recv() (*Heartbeat, error) + grpc.ServerStream +} + +type seaweedSendHeartbeatServer struct { + grpc.ServerStream +} + +func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) { + m := new(Heartbeat) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _Seaweed_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedServer).KeepConnected(&seaweedKeepConnectedServer{stream}) +} + +type Seaweed_KeepConnectedServer interface { + Send(*KeepConnectedResponse) error + Recv() (*KeepConnectedRequest, error) + grpc.ServerStream +} + +type seaweedKeepConnectedServer struct { + grpc.ServerStream +} + +func (x *seaweedKeepConnectedServer) Send(m *KeepConnectedResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedKeepConnectedServer) Recv() (*KeepConnectedRequest, error) { + m := new(KeepConnectedRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _Seaweed_LookupVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).LookupVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/LookupVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).LookupVolume(ctx, req.(*LookupVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_Assign_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AssignRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).Assign(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/Assign", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).Assign(ctx, req.(*AssignRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).Statistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/Statistics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).Statistics(ctx, req.(*StatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_CollectionList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CollectionListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).CollectionList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/CollectionList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).CollectionList(ctx, req.(*CollectionListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_CollectionDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CollectionDeleteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).CollectionDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/CollectionDelete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).CollectionDelete(ctx, req.(*CollectionDeleteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_VolumeList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).VolumeList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/VolumeList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).VolumeList(ctx, req.(*VolumeListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_LookupEcVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupEcVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).LookupEcVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/LookupEcVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).LookupEcVolume(ctx, req.(*LookupEcVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_VacuumVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VacuumVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).VacuumVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/VacuumVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).VacuumVolume(ctx, req.(*VacuumVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_GetMasterConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetMasterConfigurationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).GetMasterConfiguration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/GetMasterConfiguration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).GetMasterConfiguration(ctx, req.(*GetMasterConfigurationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_ListClusterNodes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListClusterNodesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).ListClusterNodes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/ListClusterNodes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).ListClusterNodes(ctx, req.(*ListClusterNodesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_LeaseAdminToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LeaseAdminTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).LeaseAdminToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/LeaseAdminToken", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).LeaseAdminToken(ctx, req.(*LeaseAdminTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_ReleaseAdminToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseAdminTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).ReleaseAdminToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/ReleaseAdminToken", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).ReleaseAdminToken(ctx, req.(*ReleaseAdminTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).Ping(ctx, req.(*PingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_RaftListClusterServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RaftListClusterServersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).RaftListClusterServers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/RaftListClusterServers", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).RaftListClusterServers(ctx, req.(*RaftListClusterServersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_RaftAddServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RaftAddServerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).RaftAddServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/RaftAddServer", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).RaftAddServer(ctx, req.(*RaftAddServerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_RaftRemoveServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RaftRemoveServerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).RaftRemoveServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/master_pb.Seaweed/RaftRemoveServer", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).RaftRemoveServer(ctx, req.(*RaftRemoveServerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Seaweed_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "master_pb.Seaweed", + HandlerType: (*SeaweedServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "LookupVolume", + Handler: _Seaweed_LookupVolume_Handler, + }, + { + MethodName: "Assign", + Handler: _Seaweed_Assign_Handler, + }, + { + MethodName: "Statistics", + Handler: _Seaweed_Statistics_Handler, + }, + { + MethodName: "CollectionList", + Handler: _Seaweed_CollectionList_Handler, + }, + { + MethodName: "CollectionDelete", + Handler: _Seaweed_CollectionDelete_Handler, + }, + { + MethodName: "VolumeList", + Handler: _Seaweed_VolumeList_Handler, + }, + { + MethodName: "LookupEcVolume", + Handler: _Seaweed_LookupEcVolume_Handler, + }, + { + MethodName: "VacuumVolume", + Handler: _Seaweed_VacuumVolume_Handler, + }, + { + MethodName: "GetMasterConfiguration", + Handler: _Seaweed_GetMasterConfiguration_Handler, + }, + { + MethodName: "ListClusterNodes", + Handler: _Seaweed_ListClusterNodes_Handler, + }, + { + MethodName: "LeaseAdminToken", + Handler: _Seaweed_LeaseAdminToken_Handler, + }, + { + MethodName: "ReleaseAdminToken", + Handler: _Seaweed_ReleaseAdminToken_Handler, + }, + { + MethodName: "Ping", + Handler: _Seaweed_Ping_Handler, + }, + { + MethodName: "RaftListClusterServers", + Handler: _Seaweed_RaftListClusterServers_Handler, + }, + { + MethodName: "RaftAddServer", + Handler: _Seaweed_RaftAddServer_Handler, + }, + { + MethodName: "RaftRemoveServer", + Handler: _Seaweed_RaftRemoveServer_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "SendHeartbeat", + Handler: _Seaweed_SendHeartbeat_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "KeepConnected", + Handler: _Seaweed_KeepConnected_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "master.proto", +} diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go index 591406347..5b9ca1ee3 100644 --- a/weed/pb/messaging_pb/messaging.pb.go +++ b/weed/pb/messaging_pb/messaging.pb.go @@ -1,17 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: messaging.proto package messaging_pb import ( - context "context" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -25,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type SubscriberMessage_InitMessage_StartPosition int32 const ( @@ -1726,328 +1717,3 @@ func file_messaging_proto_init() { file_messaging_proto_goTypes = nil file_messaging_proto_depIdxs = nil } - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// SeaweedMessagingClient is the client API for SeaweedMessaging service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SeaweedMessagingClient interface { - Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) - Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) - DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) - ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) - GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) - FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) -} - -type seaweedMessagingClient struct { - cc grpc.ClientConnInterface -} - -func NewSeaweedMessagingClient(cc grpc.ClientConnInterface) SeaweedMessagingClient { - return &seaweedMessagingClient{cc} -} - -func (c *seaweedMessagingClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[0], "/messaging_pb.SeaweedMessaging/Subscribe", opts...) - if err != nil { - return nil, err - } - x := &seaweedMessagingSubscribeClient{stream} - return x, nil -} - -type SeaweedMessaging_SubscribeClient interface { - Send(*SubscriberMessage) error - Recv() (*BrokerMessage, error) - grpc.ClientStream -} - -type seaweedMessagingSubscribeClient struct { - grpc.ClientStream -} - -func (x *seaweedMessagingSubscribeClient) Send(m *SubscriberMessage) error { - return x.ClientStream.SendMsg(m) -} - -func (x *seaweedMessagingSubscribeClient) Recv() (*BrokerMessage, error) { - m := new(BrokerMessage) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedMessagingClient) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) { - stream, err := c.cc.NewStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[1], "/messaging_pb.SeaweedMessaging/Publish", opts...) - if err != nil { - return nil, err - } - x := &seaweedMessagingPublishClient{stream} - return x, nil -} - -type SeaweedMessaging_PublishClient interface { - Send(*PublishRequest) error - Recv() (*PublishResponse, error) - grpc.ClientStream -} - -type seaweedMessagingPublishClient struct { - grpc.ClientStream -} - -func (x *seaweedMessagingPublishClient) Send(m *PublishRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) { - m := new(PublishResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *seaweedMessagingClient) DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) { - out := new(DeleteTopicResponse) - err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/DeleteTopic", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) { - out := new(ConfigureTopicResponse) - err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ConfigureTopic", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) { - out := new(GetTopicConfigurationResponse) - err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *seaweedMessagingClient) FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) { - out := new(FindBrokerResponse) - err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/FindBroker", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SeaweedMessagingServer is the server API for SeaweedMessaging service. -type SeaweedMessagingServer interface { - Subscribe(SeaweedMessaging_SubscribeServer) error - Publish(SeaweedMessaging_PublishServer) error - DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) - ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) - GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) - FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) -} - -// UnimplementedSeaweedMessagingServer can be embedded to have forward compatible implementations. -type UnimplementedSeaweedMessagingServer struct { -} - -func (*UnimplementedSeaweedMessagingServer) Subscribe(SeaweedMessaging_SubscribeServer) error { - return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") -} -func (*UnimplementedSeaweedMessagingServer) Publish(SeaweedMessaging_PublishServer) error { - return status.Errorf(codes.Unimplemented, "method Publish not implemented") -} -func (*UnimplementedSeaweedMessagingServer) DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteTopic not implemented") -} -func (*UnimplementedSeaweedMessagingServer) ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ConfigureTopic not implemented") -} -func (*UnimplementedSeaweedMessagingServer) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetTopicConfiguration not implemented") -} -func (*UnimplementedSeaweedMessagingServer) FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method FindBroker not implemented") -} - -func RegisterSeaweedMessagingServer(s *grpc.Server, srv SeaweedMessagingServer) { - s.RegisterService(&_SeaweedMessaging_serviceDesc, srv) -} - -func _SeaweedMessaging_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SeaweedMessagingServer).Subscribe(&seaweedMessagingSubscribeServer{stream}) -} - -type SeaweedMessaging_SubscribeServer interface { - Send(*BrokerMessage) error - Recv() (*SubscriberMessage, error) - grpc.ServerStream -} - -type seaweedMessagingSubscribeServer struct { - grpc.ServerStream -} - -func (x *seaweedMessagingSubscribeServer) Send(m *BrokerMessage) error { - return x.ServerStream.SendMsg(m) -} - -func (x *seaweedMessagingSubscribeServer) Recv() (*SubscriberMessage, error) { - m := new(SubscriberMessage) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream}) -} - -type SeaweedMessaging_PublishServer interface { - Send(*PublishResponse) error - Recv() (*PublishRequest, error) - grpc.ServerStream -} - -type seaweedMessagingPublishServer struct { - grpc.ServerStream -} - -func (x *seaweedMessagingPublishServer) Send(m *PublishResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) { - m := new(PublishRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _SeaweedMessaging_DeleteTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteTopicRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedMessagingServer).DeleteTopic(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/messaging_pb.SeaweedMessaging/DeleteTopic", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedMessagingServer).DeleteTopic(ctx, req.(*DeleteTopicRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ConfigureTopicRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/messaging_pb.SeaweedMessaging/ConfigureTopic", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetTopicConfigurationRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _SeaweedMessaging_FindBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(FindBrokerRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedMessagingServer).FindBroker(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/messaging_pb.SeaweedMessaging/FindBroker", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedMessagingServer).FindBroker(ctx, req.(*FindBrokerRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ - ServiceName: "messaging_pb.SeaweedMessaging", - HandlerType: (*SeaweedMessagingServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "DeleteTopic", - Handler: _SeaweedMessaging_DeleteTopic_Handler, - }, - { - MethodName: "ConfigureTopic", - Handler: _SeaweedMessaging_ConfigureTopic_Handler, - }, - { - MethodName: "GetTopicConfiguration", - Handler: _SeaweedMessaging_GetTopicConfiguration_Handler, - }, - { - MethodName: "FindBroker", - Handler: _SeaweedMessaging_FindBroker_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Subscribe", - Handler: _SeaweedMessaging_Subscribe_Handler, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "Publish", - Handler: _SeaweedMessaging_Publish_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "messaging.proto", -} diff --git a/weed/pb/messaging_pb/messaging_grpc.pb.go b/weed/pb/messaging_pb/messaging_grpc.pb.go new file mode 100644 index 000000000..234cffa95 --- /dev/null +++ b/weed/pb/messaging_pb/messaging_grpc.pb.go @@ -0,0 +1,346 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package messaging_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedMessagingClient is the client API for SeaweedMessaging service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedMessagingClient interface { + Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) + Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) + DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) + ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) + GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) + FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) +} + +type seaweedMessagingClient struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedMessagingClient(cc grpc.ClientConnInterface) SeaweedMessagingClient { + return &seaweedMessagingClient{cc} +} + +func (c *seaweedMessagingClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[0], "/messaging_pb.SeaweedMessaging/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &seaweedMessagingSubscribeClient{stream} + return x, nil +} + +type SeaweedMessaging_SubscribeClient interface { + Send(*SubscriberMessage) error + Recv() (*BrokerMessage, error) + grpc.ClientStream +} + +type seaweedMessagingSubscribeClient struct { + grpc.ClientStream +} + +func (x *seaweedMessagingSubscribeClient) Send(m *SubscriberMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedMessagingSubscribeClient) Recv() (*BrokerMessage, error) { + m := new(BrokerMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedMessagingClient) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedMessaging_ServiceDesc.Streams[1], "/messaging_pb.SeaweedMessaging/Publish", opts...) + if err != nil { + return nil, err + } + x := &seaweedMessagingPublishClient{stream} + return x, nil +} + +type SeaweedMessaging_PublishClient interface { + Send(*PublishRequest) error + Recv() (*PublishResponse, error) + grpc.ClientStream +} + +type seaweedMessagingPublishClient struct { + grpc.ClientStream +} + +func (x *seaweedMessagingPublishClient) Send(m *PublishRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) { + m := new(PublishResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *seaweedMessagingClient) DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) { + out := new(DeleteTopicResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/DeleteTopic", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) { + out := new(ConfigureTopicResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ConfigureTopic", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) { + out := new(GetTopicConfigurationResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedMessagingClient) FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) { + out := new(FindBrokerResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/FindBroker", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SeaweedMessagingServer is the server API for SeaweedMessaging service. +// All implementations must embed UnimplementedSeaweedMessagingServer +// for forward compatibility +type SeaweedMessagingServer interface { + Subscribe(SeaweedMessaging_SubscribeServer) error + Publish(SeaweedMessaging_PublishServer) error + DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) + ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) + GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) + FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) + mustEmbedUnimplementedSeaweedMessagingServer() +} + +// UnimplementedSeaweedMessagingServer must be embedded to have forward compatible implementations. +type UnimplementedSeaweedMessagingServer struct { +} + +func (UnimplementedSeaweedMessagingServer) Subscribe(SeaweedMessaging_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedSeaweedMessagingServer) Publish(SeaweedMessaging_PublishServer) error { + return status.Errorf(codes.Unimplemented, "method Publish not implemented") +} +func (UnimplementedSeaweedMessagingServer) DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteTopic not implemented") +} +func (UnimplementedSeaweedMessagingServer) ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ConfigureTopic not implemented") +} +func (UnimplementedSeaweedMessagingServer) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTopicConfiguration not implemented") +} +func (UnimplementedSeaweedMessagingServer) FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method FindBroker not implemented") +} +func (UnimplementedSeaweedMessagingServer) mustEmbedUnimplementedSeaweedMessagingServer() {} + +// UnsafeSeaweedMessagingServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedMessagingServer will +// result in compilation errors. +type UnsafeSeaweedMessagingServer interface { + mustEmbedUnimplementedSeaweedMessagingServer() +} + +func RegisterSeaweedMessagingServer(s grpc.ServiceRegistrar, srv SeaweedMessagingServer) { + s.RegisterService(&SeaweedMessaging_ServiceDesc, srv) +} + +func _SeaweedMessaging_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedMessagingServer).Subscribe(&seaweedMessagingSubscribeServer{stream}) +} + +type SeaweedMessaging_SubscribeServer interface { + Send(*BrokerMessage) error + Recv() (*SubscriberMessage, error) + grpc.ServerStream +} + +type seaweedMessagingSubscribeServer struct { + grpc.ServerStream +} + +func (x *seaweedMessagingSubscribeServer) Send(m *BrokerMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedMessagingSubscribeServer) Recv() (*SubscriberMessage, error) { + m := new(SubscriberMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream}) +} + +type SeaweedMessaging_PublishServer interface { + Send(*PublishResponse) error + Recv() (*PublishRequest, error) + grpc.ServerStream +} + +type seaweedMessagingPublishServer struct { + grpc.ServerStream +} + +func (x *seaweedMessagingPublishServer) Send(m *PublishResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) { + m := new(PublishRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _SeaweedMessaging_DeleteTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteTopicRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).DeleteTopic(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/DeleteTopic", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).DeleteTopic(ctx, req.(*DeleteTopicRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConfigureTopicRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/ConfigureTopic", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetTopicConfigurationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedMessaging_FindBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FindBrokerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).FindBroker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/FindBroker", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).FindBroker(ctx, req.(*FindBrokerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SeaweedMessaging_ServiceDesc is the grpc.ServiceDesc for SeaweedMessaging service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "messaging_pb.SeaweedMessaging", + HandlerType: (*SeaweedMessagingServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "DeleteTopic", + Handler: _SeaweedMessaging_DeleteTopic_Handler, + }, + { + MethodName: "ConfigureTopic", + Handler: _SeaweedMessaging_ConfigureTopic_Handler, + }, + { + MethodName: "GetTopicConfiguration", + Handler: _SeaweedMessaging_GetTopicConfiguration_Handler, + }, + { + MethodName: "FindBroker", + Handler: _SeaweedMessaging_FindBroker_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _SeaweedMessaging_Subscribe_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "Publish", + Handler: _SeaweedMessaging_Publish_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "messaging.proto", +} diff --git a/weed/pb/mount.proto b/weed/pb/mount.proto new file mode 100644 index 000000000..ec0847f12 --- /dev/null +++ b/weed/pb/mount.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package messaging_pb; + +option go_package = "github.com/chrislusf/seaweedfs/weed/pb/mount_pb"; +option java_package = "seaweedfs.client"; +option java_outer_classname = "MountProto"; + +////////////////////////////////////////////////// + +service SeaweedMount { + + rpc Configure (ConfigureRequest) returns (ConfigureResponse) { + } + +} + +////////////////////////////////////////////////// + +message ConfigureRequest { + int64 collection_capacity = 1; +} + +message ConfigureResponse { +} diff --git a/weed/pb/mount_pb/mount.pb.go b/weed/pb/mount_pb/mount.pb.go new file mode 100644 index 000000000..cbaf533fe --- /dev/null +++ b/weed/pb/mount_pb/mount.pb.go @@ -0,0 +1,208 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.17.3 +// source: mount.proto + +package mount_pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ConfigureRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CollectionCapacity int64 `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"` +} + +func (x *ConfigureRequest) Reset() { + *x = ConfigureRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_mount_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConfigureRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfigureRequest) ProtoMessage() {} + +func (x *ConfigureRequest) ProtoReflect() protoreflect.Message { + mi := &file_mount_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfigureRequest.ProtoReflect.Descriptor instead. +func (*ConfigureRequest) Descriptor() ([]byte, []int) { + return file_mount_proto_rawDescGZIP(), []int{0} +} + +func (x *ConfigureRequest) GetCollectionCapacity() int64 { + if x != nil { + return x.CollectionCapacity + } + return 0 +} + +type ConfigureResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ConfigureResponse) Reset() { + *x = ConfigureResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_mount_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConfigureResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfigureResponse) ProtoMessage() {} + +func (x *ConfigureResponse) ProtoReflect() protoreflect.Message { + mi := &file_mount_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfigureResponse.ProtoReflect.Descriptor instead. +func (*ConfigureResponse) Descriptor() ([]byte, []int) { + return file_mount_proto_rawDescGZIP(), []int{1} +} + +var File_mount_proto protoreflect.FileDescriptor + +var file_mount_proto_rawDesc = []byte{ + 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x43, 0x0a, 0x10, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x61, + 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, + 0x22, 0x13, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x5e, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, + 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x4e, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, + 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, + 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, + 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, + 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x6f, + 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_mount_proto_rawDescOnce sync.Once + file_mount_proto_rawDescData = file_mount_proto_rawDesc +) + +func file_mount_proto_rawDescGZIP() []byte { + file_mount_proto_rawDescOnce.Do(func() { + file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(file_mount_proto_rawDescData) + }) + return file_mount_proto_rawDescData +} + +var file_mount_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_mount_proto_goTypes = []interface{}{ + (*ConfigureRequest)(nil), // 0: messaging_pb.ConfigureRequest + (*ConfigureResponse)(nil), // 1: messaging_pb.ConfigureResponse +} +var file_mount_proto_depIdxs = []int32{ + 0, // 0: messaging_pb.SeaweedMount.Configure:input_type -> messaging_pb.ConfigureRequest + 1, // 1: messaging_pb.SeaweedMount.Configure:output_type -> messaging_pb.ConfigureResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_mount_proto_init() } +func file_mount_proto_init() { + if File_mount_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_mount_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConfigureRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_mount_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConfigureResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_mount_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_mount_proto_goTypes, + DependencyIndexes: file_mount_proto_depIdxs, + MessageInfos: file_mount_proto_msgTypes, + }.Build() + File_mount_proto = out.File + file_mount_proto_rawDesc = nil + file_mount_proto_goTypes = nil + file_mount_proto_depIdxs = nil +} diff --git a/weed/pb/mount_pb/mount_grpc.pb.go b/weed/pb/mount_pb/mount_grpc.pb.go new file mode 100644 index 000000000..41737aa21 --- /dev/null +++ b/weed/pb/mount_pb/mount_grpc.pb.go @@ -0,0 +1,101 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package mount_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedMountClient is the client API for SeaweedMount service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedMountClient interface { + Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) +} + +type seaweedMountClient struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedMountClient(cc grpc.ClientConnInterface) SeaweedMountClient { + return &seaweedMountClient{cc} +} + +func (c *seaweedMountClient) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) { + out := new(ConfigureResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMount/Configure", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SeaweedMountServer is the server API for SeaweedMount service. +// All implementations must embed UnimplementedSeaweedMountServer +// for forward compatibility +type SeaweedMountServer interface { + Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) + mustEmbedUnimplementedSeaweedMountServer() +} + +// UnimplementedSeaweedMountServer must be embedded to have forward compatible implementations. +type UnimplementedSeaweedMountServer struct { +} + +func (UnimplementedSeaweedMountServer) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") +} +func (UnimplementedSeaweedMountServer) mustEmbedUnimplementedSeaweedMountServer() {} + +// UnsafeSeaweedMountServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedMountServer will +// result in compilation errors. +type UnsafeSeaweedMountServer interface { + mustEmbedUnimplementedSeaweedMountServer() +} + +func RegisterSeaweedMountServer(s grpc.ServiceRegistrar, srv SeaweedMountServer) { + s.RegisterService(&SeaweedMount_ServiceDesc, srv) +} + +func _SeaweedMount_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ConfigureRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMountServer).Configure(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMount/Configure", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMountServer).Configure(ctx, req.(*ConfigureRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SeaweedMount_ServiceDesc is the grpc.ServiceDesc for SeaweedMount service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SeaweedMount_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "messaging_pb.SeaweedMount", + HandlerType: (*SeaweedMountServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Configure", + Handler: _SeaweedMount_Configure_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "mount.proto", +} diff --git a/weed/pb/proto_read_write_test.go b/weed/pb/proto_read_write_test.go index 7f6444ab5..d6b2faaef 100644 --- a/weed/pb/proto_read_write_test.go +++ b/weed/pb/proto_read_write_test.go @@ -30,7 +30,7 @@ func TestJsonpMarshalUnmarshal(t *testing.T) { rawJson := `{ "backendType":"aws", "backendId":"temp", - "FileSize":12 + "fileSize":12 }` tv1 := &volume_server_pb.RemoteFile{} diff --git a/weed/pb/remote.proto b/weed/pb/remote.proto index 1f1c37343..13f7a878b 100644 --- a/weed/pb/remote.proto +++ b/weed/pb/remote.proto @@ -18,8 +18,10 @@ message RemoteConf { string s3_endpoint = 7; string s3_storage_class = 8; bool s3_force_path_style = 9; + bool s3_v4_signature = 11; string gcs_google_application_credentials = 10; + string gcs_project_id = 12; string azure_account_name = 15; string azure_account_key = 16; @@ -47,15 +49,23 @@ message RemoteConf { string wasabi_endpoint = 42; string wasabi_region = 43; - repeated string hdfs_namenodes = 50; - string hdfs_username = 51; - string hdfs_service_principal_name = 52; - string hdfs_data_transfer_protection = 53; + string filebase_access_key = 60; + string filebase_secret_key = 61; + string filebase_endpoint = 62; + string storj_access_key = 65; + string storj_secret_key = 66; + string storj_endpoint = 67; + + string contabo_access_key = 68; + string contabo_secret_key = 69; + string contabo_endpoint = 70; + string contabo_region = 71; } message RemoteStorageMapping { map mappings = 1; + string primary_bucket_storage_name = 2; } message RemoteStorageLocation { string name = 1; diff --git a/weed/pb/remote_pb/remote.pb.go b/weed/pb/remote_pb/remote.pb.go index cc513fe26..f746333ff 100644 --- a/weed/pb/remote_pb/remote.pb.go +++ b/weed/pb/remote_pb/remote.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: remote.proto package remote_pb import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - ///////////////////////// // Remote Storage related ///////////////////////// @@ -41,7 +36,9 @@ type RemoteConf struct { S3Endpoint string `protobuf:"bytes,7,opt,name=s3_endpoint,json=s3Endpoint,proto3" json:"s3_endpoint,omitempty"` S3StorageClass string `protobuf:"bytes,8,opt,name=s3_storage_class,json=s3StorageClass,proto3" json:"s3_storage_class,omitempty"` S3ForcePathStyle bool `protobuf:"varint,9,opt,name=s3_force_path_style,json=s3ForcePathStyle,proto3" json:"s3_force_path_style,omitempty"` + S3V4Signature bool `protobuf:"varint,11,opt,name=s3_v4_signature,json=s3V4Signature,proto3" json:"s3_v4_signature,omitempty"` GcsGoogleApplicationCredentials string `protobuf:"bytes,10,opt,name=gcs_google_application_credentials,json=gcsGoogleApplicationCredentials,proto3" json:"gcs_google_application_credentials,omitempty"` + GcsProjectId string `protobuf:"bytes,12,opt,name=gcs_project_id,json=gcsProjectId,proto3" json:"gcs_project_id,omitempty"` AzureAccountName string `protobuf:"bytes,15,opt,name=azure_account_name,json=azureAccountName,proto3" json:"azure_account_name,omitempty"` AzureAccountKey string `protobuf:"bytes,16,opt,name=azure_account_key,json=azureAccountKey,proto3" json:"azure_account_key,omitempty"` BackblazeKeyId string `protobuf:"bytes,20,opt,name=backblaze_key_id,json=backblazeKeyId,proto3" json:"backblaze_key_id,omitempty"` @@ -66,6 +63,16 @@ type RemoteConf struct { HdfsUsername string `protobuf:"bytes,51,opt,name=hdfs_username,json=hdfsUsername,proto3" json:"hdfs_username,omitempty"` HdfsServicePrincipalName string `protobuf:"bytes,52,opt,name=hdfs_service_principal_name,json=hdfsServicePrincipalName,proto3" json:"hdfs_service_principal_name,omitempty"` HdfsDataTransferProtection string `protobuf:"bytes,53,opt,name=hdfs_data_transfer_protection,json=hdfsDataTransferProtection,proto3" json:"hdfs_data_transfer_protection,omitempty"` + FilebaseAccessKey string `protobuf:"bytes,60,opt,name=filebase_access_key,json=filebaseAccessKey,proto3" json:"filebase_access_key,omitempty"` + FilebaseSecretKey string `protobuf:"bytes,61,opt,name=filebase_secret_key,json=filebaseSecretKey,proto3" json:"filebase_secret_key,omitempty"` + FilebaseEndpoint string `protobuf:"bytes,62,opt,name=filebase_endpoint,json=filebaseEndpoint,proto3" json:"filebase_endpoint,omitempty"` + StorjAccessKey string `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"` + StorjSecretKey string `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"` + StorjEndpoint string `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"` + ContaboAccessKey string `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"` + ContaboSecretKey string `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"` + ContaboEndpoint string `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"` + ContaboRegion string `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"` } func (x *RemoteConf) Reset() { @@ -156,6 +163,13 @@ func (x *RemoteConf) GetS3ForcePathStyle() bool { return false } +func (x *RemoteConf) GetS3V4Signature() bool { + if x != nil { + return x.S3V4Signature + } + return false +} + func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string { if x != nil { return x.GcsGoogleApplicationCredentials @@ -163,6 +177,13 @@ func (x *RemoteConf) GetGcsGoogleApplicationCredentials() string { return "" } +func (x *RemoteConf) GetGcsProjectId() string { + if x != nil { + return x.GcsProjectId + } + return "" +} + func (x *RemoteConf) GetAzureAccountName() string { if x != nil { return x.AzureAccountName @@ -331,12 +352,83 @@ func (x *RemoteConf) GetHdfsDataTransferProtection() string { return "" } +func (x *RemoteConf) GetFilebaseAccessKey() string { + if x != nil { + return x.FilebaseAccessKey + } + return "" +} + +func (x *RemoteConf) GetFilebaseSecretKey() string { + if x != nil { + return x.FilebaseSecretKey + } + return "" +} + +func (x *RemoteConf) GetFilebaseEndpoint() string { + if x != nil { + return x.FilebaseEndpoint + } + return "" +} + +func (x *RemoteConf) GetStorjAccessKey() string { + if x != nil { + return x.StorjAccessKey + } + return "" +} + +func (x *RemoteConf) GetStorjSecretKey() string { + if x != nil { + return x.StorjSecretKey + } + return "" +} + +func (x *RemoteConf) GetStorjEndpoint() string { + if x != nil { + return x.StorjEndpoint + } + return "" +} + +func (x *RemoteConf) GetContaboAccessKey() string { + if x != nil { + return x.ContaboAccessKey + } + return "" +} + +func (x *RemoteConf) GetContaboSecretKey() string { + if x != nil { + return x.ContaboSecretKey + } + return "" +} + +func (x *RemoteConf) GetContaboEndpoint() string { + if x != nil { + return x.ContaboEndpoint + } + return "" +} + +func (x *RemoteConf) GetContaboRegion() string { + if x != nil { + return x.ContaboRegion + } + return "" +} + type RemoteStorageMapping struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Mappings map[string]*RemoteStorageLocation `protobuf:"bytes,1,rep,name=mappings,proto3" json:"mappings,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + PrimaryBucketStorageName string `protobuf:"bytes,2,opt,name=primary_bucket_storage_name,json=primaryBucketStorageName,proto3" json:"primary_bucket_storage_name,omitempty"` } func (x *RemoteStorageMapping) Reset() { @@ -378,6 +470,13 @@ func (x *RemoteStorageMapping) GetMappings() map[string]*RemoteStorageLocation { return nil } +func (x *RemoteStorageMapping) GetPrimaryBucketStorageName() string { + if x != nil { + return x.PrimaryBucketStorageName + } + return "" +} + type RemoteStorageLocation struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -445,7 +544,7 @@ var File_remote_proto protoreflect.FileDescriptor var file_remote_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x8c, 0x0b, 0x0a, 0x0a, 0x52, 0x65, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x90, 0x0f, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, @@ -463,101 +562,138 @@ var file_remote_proto_rawDesc = []byte{ 0x12, 0x2d, 0x0a, 0x13, 0x73, 0x33, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, 0x33, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x53, 0x74, 0x79, 0x6c, 0x65, 0x12, - 0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, - 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12, 0x2c, 0x0a, 0x12, - 0x61, 0x7a, 0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, - 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, - 0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, - 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, - 0x61, 0x7a, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, - 0x12, 0x3a, 0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, - 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, - 0x61, 0x7a, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x61, - 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, - 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, - 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, - 0x69, 0x79, 0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, - 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, - 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, - 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, - 0x12, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, - 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, - 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x20, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, - 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, - 0x75, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, - 0x69, 0x64, 0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, - 0x6e, 0x18, 0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, - 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, - 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, - 0x61, 0x62, 0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, - 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x2a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, - 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, - 0x73, 0x61, 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, - 0x66, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, - 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, - 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, - 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, - 0x66, 0x73, 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, - 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, - 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x5d, 0x0a, 0x0d, - 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, - 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, - 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, - 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x26, 0x0a, 0x0f, 0x73, 0x33, 0x5f, 0x76, 0x34, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x73, 0x33, 0x56, 0x34, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x4b, 0x0a, 0x22, 0x67, 0x63, 0x73, 0x5f, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x1f, 0x67, 0x63, 0x73, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x41, 0x70, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x67, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x67, 0x63, + 0x73, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x7a, + 0x75, 0x72, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x7a, 0x75, 0x72, + 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, + 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x12, 0x3a, + 0x0a, 0x19, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x15, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x17, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x41, 0x70, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x12, 0x62, 0x61, + 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x61, 0x63, 0x6b, 0x62, 0x6c, 0x61, 0x7a, + 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, + 0x79, 0x75, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x19, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x41, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, + 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, + 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x6c, 0x69, 0x79, + 0x75, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, + 0x69, 0x79, 0x75, 0x6e, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x1c, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x61, 0x6c, 0x69, 0x79, 0x75, 0x6e, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, + 0x2a, 0x0a, 0x11, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, + 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x74, + 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, + 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x6e, + 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x20, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x61, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x23, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x62, 0x61, 0x69, 0x64, 0x75, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28, + 0x0a, 0x10, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x62, 0x61, 0x69, 0x64, 0x75, 0x53, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x61, 0x69, 0x64, + 0x75, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x25, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x62, 0x61, 0x69, 0x64, 0x75, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, + 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x69, 0x64, 0x75, 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, + 0x26, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x69, 0x64, 0x75, 0x52, 0x65, 0x67, 0x69, + 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, + 0x61, 0x73, 0x61, 0x62, 0x69, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2a, + 0x0a, 0x11, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x61, 0x73, 0x61, 0x62, + 0x69, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x61, + 0x73, 0x61, 0x62, 0x69, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x2a, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x61, 0x73, 0x61, 0x62, 0x69, 0x5f, 0x72, 0x65, + 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x61, 0x73, 0x61, + 0x62, 0x69, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x68, 0x64, 0x66, 0x73, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x32, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, + 0x23, 0x0a, 0x0d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x33, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x64, 0x66, 0x73, 0x55, 0x73, 0x65, 0x72, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x1b, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x34, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x68, 0x64, 0x66, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, 0x6c, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x1d, 0x68, 0x64, 0x66, 0x73, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x35, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x68, 0x64, 0x66, 0x73, + 0x44, 0x61, 0x74, 0x61, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, + 0x73, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3c, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x41, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, + 0x73, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x3d, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x53, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, + 0x73, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x3e, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x62, 0x61, 0x73, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x41, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, + 0x74, 0x6f, 0x72, 0x6a, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, + 0x10, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, + 0x79, 0x18, 0x42, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, + 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2c, + 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x44, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x62, 0x6f, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x12, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x45, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, + 0x6f, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x46, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x45, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, + 0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x47, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xff, 0x01, 0x0a, + 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, + 0x12, 0x3d, 0x0a, 0x1b, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x42, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x1a, + 0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, + 0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, + 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, + 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/weed/pb/s3.proto b/weed/pb/s3.proto new file mode 100644 index 000000000..45a877fac --- /dev/null +++ b/weed/pb/s3.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package messaging_pb; + +option go_package = "github.com/chrislusf/seaweedfs/weed/pb/s3_pb"; +option java_package = "seaweedfs.client"; +option java_outer_classname = "S3Proto"; + +////////////////////////////////////////////////// + +service SeaweedS3 { + + rpc Configure (S3ConfigureRequest) returns (S3ConfigureResponse) { + } + +} + +////////////////////////////////////////////////// + +message S3ConfigureRequest { + bytes s3_configuration_file_content = 1; +} + +message S3ConfigureResponse { +} + +message S3CircuitBreakerConfig { + S3CircuitBreakerOptions global=1; + map buckets= 2; +} + +message S3CircuitBreakerOptions { + bool enabled=1; + map actions = 2; +} diff --git a/weed/pb/s3_pb/s3.pb.go b/weed/pb/s3_pb/s3.pb.go new file mode 100644 index 000000000..c1bd23556 --- /dev/null +++ b/weed/pb/s3_pb/s3.pb.go @@ -0,0 +1,380 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.21.1 +// source: s3.proto + +package s3_pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type S3ConfigureRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + S3ConfigurationFileContent []byte `protobuf:"bytes,1,opt,name=s3_configuration_file_content,json=s3ConfigurationFileContent,proto3" json:"s3_configuration_file_content,omitempty"` +} + +func (x *S3ConfigureRequest) Reset() { + *x = S3ConfigureRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_s3_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *S3ConfigureRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3ConfigureRequest) ProtoMessage() {} + +func (x *S3ConfigureRequest) ProtoReflect() protoreflect.Message { + mi := &file_s3_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3ConfigureRequest.ProtoReflect.Descriptor instead. +func (*S3ConfigureRequest) Descriptor() ([]byte, []int) { + return file_s3_proto_rawDescGZIP(), []int{0} +} + +func (x *S3ConfigureRequest) GetS3ConfigurationFileContent() []byte { + if x != nil { + return x.S3ConfigurationFileContent + } + return nil +} + +type S3ConfigureResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *S3ConfigureResponse) Reset() { + *x = S3ConfigureResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_s3_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *S3ConfigureResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3ConfigureResponse) ProtoMessage() {} + +func (x *S3ConfigureResponse) ProtoReflect() protoreflect.Message { + mi := &file_s3_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3ConfigureResponse.ProtoReflect.Descriptor instead. +func (*S3ConfigureResponse) Descriptor() ([]byte, []int) { + return file_s3_proto_rawDescGZIP(), []int{1} +} + +type S3CircuitBreakerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Global *S3CircuitBreakerOptions `protobuf:"bytes,1,opt,name=global,proto3" json:"global,omitempty"` + Buckets map[string]*S3CircuitBreakerOptions `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *S3CircuitBreakerConfig) Reset() { + *x = S3CircuitBreakerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_s3_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *S3CircuitBreakerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3CircuitBreakerConfig) ProtoMessage() {} + +func (x *S3CircuitBreakerConfig) ProtoReflect() protoreflect.Message { + mi := &file_s3_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3CircuitBreakerConfig.ProtoReflect.Descriptor instead. +func (*S3CircuitBreakerConfig) Descriptor() ([]byte, []int) { + return file_s3_proto_rawDescGZIP(), []int{2} +} + +func (x *S3CircuitBreakerConfig) GetGlobal() *S3CircuitBreakerOptions { + if x != nil { + return x.Global + } + return nil +} + +func (x *S3CircuitBreakerConfig) GetBuckets() map[string]*S3CircuitBreakerOptions { + if x != nil { + return x.Buckets + } + return nil +} + +type S3CircuitBreakerOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + Actions map[string]int64 `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +} + +func (x *S3CircuitBreakerOptions) Reset() { + *x = S3CircuitBreakerOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_s3_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *S3CircuitBreakerOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3CircuitBreakerOptions) ProtoMessage() {} + +func (x *S3CircuitBreakerOptions) ProtoReflect() protoreflect.Message { + mi := &file_s3_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3CircuitBreakerOptions.ProtoReflect.Descriptor instead. +func (*S3CircuitBreakerOptions) Descriptor() ([]byte, []int) { + return file_s3_proto_rawDescGZIP(), []int{3} +} + +func (x *S3CircuitBreakerOptions) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + +func (x *S3CircuitBreakerOptions) GetActions() map[string]int64 { + if x != nil { + return x.Actions + } + return nil +} + +var File_s3_proto protoreflect.FileDescriptor + +var file_s3_proto_rawDesc = []byte{ + 0x0a, 0x08, 0x73, 0x33, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x57, 0x0a, 0x12, 0x53, 0x33, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, + 0x0a, 0x1d, 0x73, 0x33, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x1a, 0x73, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x33, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x87, 0x02, 0x0a, 0x16, 0x53, 0x33, 0x43, + 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x3d, 0x0a, 0x06, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, + 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x06, 0x67, 0x6c, 0x6f, 0x62, + 0x61, 0x6c, 0x12, 0x4b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, + 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x1a, + 0x61, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x3b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, + 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xbd, 0x01, 0x0a, 0x17, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, + 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x69, 0x72, 0x63, 0x75, + 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x32, 0x5f, 0x0a, 0x09, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12, + 0x52, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x20, 0x2e, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, + 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x33, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, + 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, + 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, + 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_s3_proto_rawDescOnce sync.Once + file_s3_proto_rawDescData = file_s3_proto_rawDesc +) + +func file_s3_proto_rawDescGZIP() []byte { + file_s3_proto_rawDescOnce.Do(func() { + file_s3_proto_rawDescData = protoimpl.X.CompressGZIP(file_s3_proto_rawDescData) + }) + return file_s3_proto_rawDescData +} + +var file_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_s3_proto_goTypes = []interface{}{ + (*S3ConfigureRequest)(nil), // 0: messaging_pb.S3ConfigureRequest + (*S3ConfigureResponse)(nil), // 1: messaging_pb.S3ConfigureResponse + (*S3CircuitBreakerConfig)(nil), // 2: messaging_pb.S3CircuitBreakerConfig + (*S3CircuitBreakerOptions)(nil), // 3: messaging_pb.S3CircuitBreakerOptions + nil, // 4: messaging_pb.S3CircuitBreakerConfig.BucketsEntry + nil, // 5: messaging_pb.S3CircuitBreakerOptions.ActionsEntry +} +var file_s3_proto_depIdxs = []int32{ + 3, // 0: messaging_pb.S3CircuitBreakerConfig.global:type_name -> messaging_pb.S3CircuitBreakerOptions + 4, // 1: messaging_pb.S3CircuitBreakerConfig.buckets:type_name -> messaging_pb.S3CircuitBreakerConfig.BucketsEntry + 5, // 2: messaging_pb.S3CircuitBreakerOptions.actions:type_name -> messaging_pb.S3CircuitBreakerOptions.ActionsEntry + 3, // 3: messaging_pb.S3CircuitBreakerConfig.BucketsEntry.value:type_name -> messaging_pb.S3CircuitBreakerOptions + 0, // 4: messaging_pb.SeaweedS3.Configure:input_type -> messaging_pb.S3ConfigureRequest + 1, // 5: messaging_pb.SeaweedS3.Configure:output_type -> messaging_pb.S3ConfigureResponse + 5, // [5:6] is the sub-list for method output_type + 4, // [4:5] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_s3_proto_init() } +func file_s3_proto_init() { + if File_s3_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_s3_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*S3ConfigureRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_s3_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*S3ConfigureResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_s3_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*S3CircuitBreakerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_s3_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*S3CircuitBreakerOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_s3_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_s3_proto_goTypes, + DependencyIndexes: file_s3_proto_depIdxs, + MessageInfos: file_s3_proto_msgTypes, + }.Build() + File_s3_proto = out.File + file_s3_proto_rawDesc = nil + file_s3_proto_goTypes = nil + file_s3_proto_depIdxs = nil +} diff --git a/weed/pb/s3_pb/s3_grpc.pb.go b/weed/pb/s3_pb/s3_grpc.pb.go new file mode 100644 index 000000000..1bc956be6 --- /dev/null +++ b/weed/pb/s3_pb/s3_grpc.pb.go @@ -0,0 +1,101 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package s3_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// SeaweedS3Client is the client API for SeaweedS3 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type SeaweedS3Client interface { + Configure(ctx context.Context, in *S3ConfigureRequest, opts ...grpc.CallOption) (*S3ConfigureResponse, error) +} + +type seaweedS3Client struct { + cc grpc.ClientConnInterface +} + +func NewSeaweedS3Client(cc grpc.ClientConnInterface) SeaweedS3Client { + return &seaweedS3Client{cc} +} + +func (c *seaweedS3Client) Configure(ctx context.Context, in *S3ConfigureRequest, opts ...grpc.CallOption) (*S3ConfigureResponse, error) { + out := new(S3ConfigureResponse) + err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedS3/Configure", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SeaweedS3Server is the server API for SeaweedS3 service. +// All implementations must embed UnimplementedSeaweedS3Server +// for forward compatibility +type SeaweedS3Server interface { + Configure(context.Context, *S3ConfigureRequest) (*S3ConfigureResponse, error) + mustEmbedUnimplementedSeaweedS3Server() +} + +// UnimplementedSeaweedS3Server must be embedded to have forward compatible implementations. +type UnimplementedSeaweedS3Server struct { +} + +func (UnimplementedSeaweedS3Server) Configure(context.Context, *S3ConfigureRequest) (*S3ConfigureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") +} +func (UnimplementedSeaweedS3Server) mustEmbedUnimplementedSeaweedS3Server() {} + +// UnsafeSeaweedS3Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SeaweedS3Server will +// result in compilation errors. +type UnsafeSeaweedS3Server interface { + mustEmbedUnimplementedSeaweedS3Server() +} + +func RegisterSeaweedS3Server(s grpc.ServiceRegistrar, srv SeaweedS3Server) { + s.RegisterService(&SeaweedS3_ServiceDesc, srv) +} + +func _SeaweedS3_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(S3ConfigureRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedS3Server).Configure(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedS3/Configure", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedS3Server).Configure(ctx, req.(*S3ConfigureRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// SeaweedS3_ServiceDesc is the grpc.ServiceDesc for SeaweedS3 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SeaweedS3_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "messaging_pb.SeaweedS3", + HandlerType: (*SeaweedS3Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Configure", + Handler: _SeaweedS3_Configure_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "s3.proto", +} diff --git a/weed/pb/server_address.go b/weed/pb/server_address.go new file mode 100644 index 000000000..a66d0d831 --- /dev/null +++ b/weed/pb/server_address.go @@ -0,0 +1,145 @@ +package pb + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "net" + "strconv" + "strings" +) + +type ServerAddress string +type ServerAddresses string + +func NewServerAddress(host string, port int, grpcPort int) ServerAddress { + if grpcPort == 0 || grpcPort == port+10000 { + return ServerAddress(util.JoinHostPort(host, port)) + } + return ServerAddress(util.JoinHostPort(host, port) + "." + strconv.Itoa(grpcPort)) +} + +func NewServerAddressWithGrpcPort(address string, grpcPort int) ServerAddress { + if grpcPort == 0 { + return ServerAddress(address) + } + _, port, _ := hostAndPort(address) + if uint64(grpcPort) == port+10000 { + return ServerAddress(address) + } + return ServerAddress(address + "." + strconv.Itoa(grpcPort)) +} + +func NewServerAddressFromDataNode(dn *master_pb.DataNodeInfo) ServerAddress { + return NewServerAddressWithGrpcPort(dn.Id, int(dn.GrpcPort)) +} + +func NewServerAddressFromLocation(dn *master_pb.Location) ServerAddress { + return NewServerAddressWithGrpcPort(dn.Url, int(dn.GrpcPort)) +} + +func (sa ServerAddress) String() string { + return sa.ToHttpAddress() +} + +func (sa ServerAddress) ToHttpAddress() string { + portsSepIndex := strings.LastIndex(string(sa), ":") + if portsSepIndex < 0 { + return string(sa) + } + if portsSepIndex+1 >= len(sa) { + return string(sa) + } + ports := string(sa[portsSepIndex+1:]) + sepIndex := strings.LastIndex(string(ports), ".") + if sepIndex >= 0 { + host := string(sa[0:portsSepIndex]) + return net.JoinHostPort(host, ports[0:sepIndex]) + } + return string(sa) +} + +func (sa ServerAddress) ToGrpcAddress() string { + portsSepIndex := strings.LastIndex(string(sa), ":") + if portsSepIndex < 0 { + return string(sa) + } + if portsSepIndex+1 >= len(sa) { + return string(sa) + } + ports := string(sa[portsSepIndex+1:]) + sepIndex := strings.LastIndex(ports, ".") + if sepIndex >= 0 { + host := string(sa[0:portsSepIndex]) + return net.JoinHostPort(host, ports[sepIndex+1:]) + } + return ServerToGrpcAddress(string(sa)) +} + +func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) { + parts := strings.Split(string(sa), ",") + for _, address := range parts { + if address != "" { + addresses = append(addresses, ServerAddress(address)) + } + } + return +} + +func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) { + addresses = make(map[string]ServerAddress) + for _, address := range sa.ToAddresses() { + addresses[string(address)] = address + } + return +} + +func (sa ServerAddresses) ToAddressStrings() (addresses []string) { + parts := strings.Split(string(sa), ",") + for _, address := range parts { + addresses = append(addresses, address) + } + return +} + +func ToAddressStrings(addresses []ServerAddress) []string { + var strings []string + for _, addr := range addresses { + strings = append(strings, string(addr)) + } + return strings +} +func ToAddressStringsFromMap(addresses map[string]ServerAddress) []string { + var strings []string + for _, addr := range addresses { + strings = append(strings, string(addr)) + } + return strings +} +func FromAddressStrings(strings []string) []ServerAddress { + var addresses []ServerAddress + for _, addr := range strings { + addresses = append(addresses, ServerAddress(addr)) + } + return addresses +} + +func ParseUrl(input string) (address ServerAddress, path string, err error) { + if !strings.HasPrefix(input, "http://") { + return "", "", fmt.Errorf("url %s needs prefix 'http://'", input) + } + input = input[7:] + pathSeparatorIndex := strings.Index(input, "/") + hostAndPorts := input + if pathSeparatorIndex > 0 { + path = input[pathSeparatorIndex:] + hostAndPorts = input[0:pathSeparatorIndex] + } + commaSeparatorIndex := strings.Index(input, ":") + if commaSeparatorIndex < 0 { + err = fmt.Errorf("port should be specified in %s", input) + return + } + address = ServerAddress(hostAndPorts) + return +} diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 87c162bf0..6b5fbe2f9 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -14,7 +14,7 @@ service VolumeServer { rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) { } - rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) { + rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) { } rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) { } @@ -47,7 +47,7 @@ service VolumeServer { } // copy the .idx .dat files, and mount this volume - rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) { + rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) { } rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) { } @@ -58,6 +58,8 @@ service VolumeServer { } rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) { } + rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) { + } rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) { } @@ -105,6 +107,10 @@ service VolumeServer { rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) { } + + rpc Ping (PingRequest) returns (PingResponse) { + } + } ////////////////////////////////////////////////// @@ -140,6 +146,7 @@ message VacuumVolumeCompactRequest { int64 preallocate = 2; } message VacuumVolumeCompactResponse { + int64 processed_bytes = 1; } message VacuumVolumeCommitRequest { @@ -249,6 +256,7 @@ message VolumeCopyRequest { } message VolumeCopyResponse { uint64 last_append_at_ns = 1; + int64 processed_bytes = 2; } message CopyFileRequest { @@ -262,6 +270,7 @@ message CopyFileRequest { } message CopyFileResponse { bytes file_content = 1; + int64 modified_ts_ns = 2; } message ReadNeedleBlobRequest { @@ -283,6 +292,16 @@ message WriteNeedleBlobRequest { message WriteNeedleBlobResponse { } +message ReadAllNeedlesRequest { + repeated uint32 volume_ids = 1; +} +message ReadAllNeedlesResponse { + uint32 volume_id = 1; + uint64 needle_id = 2; + uint32 cookie = 3; + bytes needle_blob = 5; +} + message VolumeTailSenderRequest { uint32 volume_id = 1; uint64 since_ns = 2; @@ -460,6 +479,9 @@ message VolumeServerStatusRequest { message VolumeServerStatusResponse { repeated DiskStatus disk_statuses = 1; MemStatus memory_status = 2; + string version = 3; + string data_center = 4; + string rack = 5; } message VolumeServerLeaveRequest { @@ -474,6 +496,13 @@ message FetchAndWriteNeedleRequest { uint32 cookie = 3; int64 offset = 4; int64 size = 5; + message Replica { + string url = 1; + string public_url = 2; + int32 grpc_port = 3; + } + repeated Replica replicas = 6; + string auth = 7; // remote conf remote_pb.RemoteConf remote_conf = 15; remote_pb.RemoteStorageLocation remote_location = 16; @@ -551,3 +580,13 @@ message VolumeNeedleStatusResponse { uint32 crc = 5; string ttl = 6; } + +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +} diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index d07acbb46..4e4690f13 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1,18 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: volume_server.proto package volume_server_pb import ( - context "context" remote_pb "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -26,10 +21,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type BatchDeleteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -402,6 +393,8 @@ type VacuumVolumeCompactResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + ProcessedBytes int64 `protobuf:"varint,1,opt,name=processed_bytes,json=processedBytes,proto3" json:"processed_bytes,omitempty"` } func (x *VacuumVolumeCompactResponse) Reset() { @@ -436,6 +429,13 @@ func (*VacuumVolumeCompactResponse) Descriptor() ([]byte, []int) { return file_volume_server_proto_rawDescGZIP(), []int{7} } +func (x *VacuumVolumeCompactResponse) GetProcessedBytes() int64 { + if x != nil { + return x.ProcessedBytes + } + return 0 +} + type VacuumVolumeCommitRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1791,6 +1791,7 @@ type VolumeCopyResponse struct { unknownFields protoimpl.UnknownFields LastAppendAtNs uint64 `protobuf:"varint,1,opt,name=last_append_at_ns,json=lastAppendAtNs,proto3" json:"last_append_at_ns,omitempty"` + ProcessedBytes int64 `protobuf:"varint,2,opt,name=processed_bytes,json=processedBytes,proto3" json:"processed_bytes,omitempty"` } func (x *VolumeCopyResponse) Reset() { @@ -1832,6 +1833,13 @@ func (x *VolumeCopyResponse) GetLastAppendAtNs() uint64 { return 0 } +func (x *VolumeCopyResponse) GetProcessedBytes() int64 { + if x != nil { + return x.ProcessedBytes + } + return 0 +} + type CopyFileRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1932,7 +1940,8 @@ type CopyFileResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` + FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` + ModifiedTsNs int64 `protobuf:"varint,2,opt,name=modified_ts_ns,json=modifiedTsNs,proto3" json:"modified_ts_ns,omitempty"` } func (x *CopyFileResponse) Reset() { @@ -1974,6 +1983,13 @@ func (x *CopyFileResponse) GetFileContent() []byte { return nil } +func (x *CopyFileResponse) GetModifiedTsNs() int64 { + if x != nil { + return x.ModifiedTsNs + } + return 0 +} + type ReadNeedleBlobRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2201,6 +2217,124 @@ func (*WriteNeedleBlobResponse) Descriptor() ([]byte, []int) { return file_volume_server_proto_rawDescGZIP(), []int{41} } +type ReadAllNeedlesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + VolumeIds []uint32 `protobuf:"varint,1,rep,packed,name=volume_ids,json=volumeIds,proto3" json:"volume_ids,omitempty"` +} + +func (x *ReadAllNeedlesRequest) Reset() { + *x = ReadAllNeedlesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_volume_server_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadAllNeedlesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadAllNeedlesRequest) ProtoMessage() {} + +func (x *ReadAllNeedlesRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[42] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadAllNeedlesRequest.ProtoReflect.Descriptor instead. +func (*ReadAllNeedlesRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{42} +} + +func (x *ReadAllNeedlesRequest) GetVolumeIds() []uint32 { + if x != nil { + return x.VolumeIds + } + return nil +} + +type ReadAllNeedlesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + NeedleId uint64 `protobuf:"varint,2,opt,name=needle_id,json=needleId,proto3" json:"needle_id,omitempty"` + Cookie uint32 `protobuf:"varint,3,opt,name=cookie,proto3" json:"cookie,omitempty"` + NeedleBlob []byte `protobuf:"bytes,5,opt,name=needle_blob,json=needleBlob,proto3" json:"needle_blob,omitempty"` +} + +func (x *ReadAllNeedlesResponse) Reset() { + *x = ReadAllNeedlesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_volume_server_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadAllNeedlesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadAllNeedlesResponse) ProtoMessage() {} + +func (x *ReadAllNeedlesResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[43] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadAllNeedlesResponse.ProtoReflect.Descriptor instead. +func (*ReadAllNeedlesResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{43} +} + +func (x *ReadAllNeedlesResponse) GetVolumeId() uint32 { + if x != nil { + return x.VolumeId + } + return 0 +} + +func (x *ReadAllNeedlesResponse) GetNeedleId() uint64 { + if x != nil { + return x.NeedleId + } + return 0 +} + +func (x *ReadAllNeedlesResponse) GetCookie() uint32 { + if x != nil { + return x.Cookie + } + return 0 +} + +func (x *ReadAllNeedlesResponse) GetNeedleBlob() []byte { + if x != nil { + return x.NeedleBlob + } + return nil +} + type VolumeTailSenderRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2214,7 +2348,7 @@ type VolumeTailSenderRequest struct { func (x *VolumeTailSenderRequest) Reset() { *x = VolumeTailSenderRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[42] + mi := &file_volume_server_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2227,7 +2361,7 @@ func (x *VolumeTailSenderRequest) String() string { func (*VolumeTailSenderRequest) ProtoMessage() {} func (x *VolumeTailSenderRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[42] + mi := &file_volume_server_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2240,7 +2374,7 @@ func (x *VolumeTailSenderRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeTailSenderRequest.ProtoReflect.Descriptor instead. func (*VolumeTailSenderRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{42} + return file_volume_server_proto_rawDescGZIP(), []int{44} } func (x *VolumeTailSenderRequest) GetVolumeId() uint32 { @@ -2277,7 +2411,7 @@ type VolumeTailSenderResponse struct { func (x *VolumeTailSenderResponse) Reset() { *x = VolumeTailSenderResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[43] + mi := &file_volume_server_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2290,7 +2424,7 @@ func (x *VolumeTailSenderResponse) String() string { func (*VolumeTailSenderResponse) ProtoMessage() {} func (x *VolumeTailSenderResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[43] + mi := &file_volume_server_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2303,7 +2437,7 @@ func (x *VolumeTailSenderResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeTailSenderResponse.ProtoReflect.Descriptor instead. func (*VolumeTailSenderResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{43} + return file_volume_server_proto_rawDescGZIP(), []int{45} } func (x *VolumeTailSenderResponse) GetNeedleHeader() []byte { @@ -2341,7 +2475,7 @@ type VolumeTailReceiverRequest struct { func (x *VolumeTailReceiverRequest) Reset() { *x = VolumeTailReceiverRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[44] + mi := &file_volume_server_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2354,7 +2488,7 @@ func (x *VolumeTailReceiverRequest) String() string { func (*VolumeTailReceiverRequest) ProtoMessage() {} func (x *VolumeTailReceiverRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[44] + mi := &file_volume_server_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2367,7 +2501,7 @@ func (x *VolumeTailReceiverRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeTailReceiverRequest.ProtoReflect.Descriptor instead. func (*VolumeTailReceiverRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{44} + return file_volume_server_proto_rawDescGZIP(), []int{46} } func (x *VolumeTailReceiverRequest) GetVolumeId() uint32 { @@ -2407,7 +2541,7 @@ type VolumeTailReceiverResponse struct { func (x *VolumeTailReceiverResponse) Reset() { *x = VolumeTailReceiverResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[45] + mi := &file_volume_server_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2420,7 +2554,7 @@ func (x *VolumeTailReceiverResponse) String() string { func (*VolumeTailReceiverResponse) ProtoMessage() {} func (x *VolumeTailReceiverResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[45] + mi := &file_volume_server_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2433,7 +2567,7 @@ func (x *VolumeTailReceiverResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeTailReceiverResponse.ProtoReflect.Descriptor instead. func (*VolumeTailReceiverResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{45} + return file_volume_server_proto_rawDescGZIP(), []int{47} } type VolumeEcShardsGenerateRequest struct { @@ -2448,7 +2582,7 @@ type VolumeEcShardsGenerateRequest struct { func (x *VolumeEcShardsGenerateRequest) Reset() { *x = VolumeEcShardsGenerateRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[46] + mi := &file_volume_server_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2461,7 +2595,7 @@ func (x *VolumeEcShardsGenerateRequest) String() string { func (*VolumeEcShardsGenerateRequest) ProtoMessage() {} func (x *VolumeEcShardsGenerateRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[46] + mi := &file_volume_server_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2474,7 +2608,7 @@ func (x *VolumeEcShardsGenerateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsGenerateRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsGenerateRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{46} + return file_volume_server_proto_rawDescGZIP(), []int{48} } func (x *VolumeEcShardsGenerateRequest) GetVolumeId() uint32 { @@ -2500,7 +2634,7 @@ type VolumeEcShardsGenerateResponse struct { func (x *VolumeEcShardsGenerateResponse) Reset() { *x = VolumeEcShardsGenerateResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[47] + mi := &file_volume_server_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2513,7 +2647,7 @@ func (x *VolumeEcShardsGenerateResponse) String() string { func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} func (x *VolumeEcShardsGenerateResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[47] + mi := &file_volume_server_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2526,7 +2660,7 @@ func (x *VolumeEcShardsGenerateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsGenerateResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{47} + return file_volume_server_proto_rawDescGZIP(), []int{49} } type VolumeEcShardsRebuildRequest struct { @@ -2541,7 +2675,7 @@ type VolumeEcShardsRebuildRequest struct { func (x *VolumeEcShardsRebuildRequest) Reset() { *x = VolumeEcShardsRebuildRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[48] + mi := &file_volume_server_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2554,7 +2688,7 @@ func (x *VolumeEcShardsRebuildRequest) String() string { func (*VolumeEcShardsRebuildRequest) ProtoMessage() {} func (x *VolumeEcShardsRebuildRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[48] + mi := &file_volume_server_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2567,7 +2701,7 @@ func (x *VolumeEcShardsRebuildRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsRebuildRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsRebuildRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{48} + return file_volume_server_proto_rawDescGZIP(), []int{50} } func (x *VolumeEcShardsRebuildRequest) GetVolumeId() uint32 { @@ -2595,7 +2729,7 @@ type VolumeEcShardsRebuildResponse struct { func (x *VolumeEcShardsRebuildResponse) Reset() { *x = VolumeEcShardsRebuildResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[49] + mi := &file_volume_server_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2608,7 +2742,7 @@ func (x *VolumeEcShardsRebuildResponse) String() string { func (*VolumeEcShardsRebuildResponse) ProtoMessage() {} func (x *VolumeEcShardsRebuildResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[49] + mi := &file_volume_server_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2621,7 +2755,7 @@ func (x *VolumeEcShardsRebuildResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsRebuildResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsRebuildResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{49} + return file_volume_server_proto_rawDescGZIP(), []int{51} } func (x *VolumeEcShardsRebuildResponse) GetRebuiltShardIds() []uint32 { @@ -2648,7 +2782,7 @@ type VolumeEcShardsCopyRequest struct { func (x *VolumeEcShardsCopyRequest) Reset() { *x = VolumeEcShardsCopyRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[50] + mi := &file_volume_server_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2661,7 +2795,7 @@ func (x *VolumeEcShardsCopyRequest) String() string { func (*VolumeEcShardsCopyRequest) ProtoMessage() {} func (x *VolumeEcShardsCopyRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[50] + mi := &file_volume_server_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2674,7 +2808,7 @@ func (x *VolumeEcShardsCopyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsCopyRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsCopyRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{50} + return file_volume_server_proto_rawDescGZIP(), []int{52} } func (x *VolumeEcShardsCopyRequest) GetVolumeId() uint32 { @@ -2735,7 +2869,7 @@ type VolumeEcShardsCopyResponse struct { func (x *VolumeEcShardsCopyResponse) Reset() { *x = VolumeEcShardsCopyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[51] + mi := &file_volume_server_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2748,7 +2882,7 @@ func (x *VolumeEcShardsCopyResponse) String() string { func (*VolumeEcShardsCopyResponse) ProtoMessage() {} func (x *VolumeEcShardsCopyResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[51] + mi := &file_volume_server_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2761,7 +2895,7 @@ func (x *VolumeEcShardsCopyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsCopyResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsCopyResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{51} + return file_volume_server_proto_rawDescGZIP(), []int{53} } type VolumeEcShardsDeleteRequest struct { @@ -2777,7 +2911,7 @@ type VolumeEcShardsDeleteRequest struct { func (x *VolumeEcShardsDeleteRequest) Reset() { *x = VolumeEcShardsDeleteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[52] + mi := &file_volume_server_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2790,7 +2924,7 @@ func (x *VolumeEcShardsDeleteRequest) String() string { func (*VolumeEcShardsDeleteRequest) ProtoMessage() {} func (x *VolumeEcShardsDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[52] + mi := &file_volume_server_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2803,7 +2937,7 @@ func (x *VolumeEcShardsDeleteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsDeleteRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsDeleteRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{52} + return file_volume_server_proto_rawDescGZIP(), []int{54} } func (x *VolumeEcShardsDeleteRequest) GetVolumeId() uint32 { @@ -2836,7 +2970,7 @@ type VolumeEcShardsDeleteResponse struct { func (x *VolumeEcShardsDeleteResponse) Reset() { *x = VolumeEcShardsDeleteResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[53] + mi := &file_volume_server_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2849,7 +2983,7 @@ func (x *VolumeEcShardsDeleteResponse) String() string { func (*VolumeEcShardsDeleteResponse) ProtoMessage() {} func (x *VolumeEcShardsDeleteResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[53] + mi := &file_volume_server_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2862,7 +2996,7 @@ func (x *VolumeEcShardsDeleteResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsDeleteResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{53} + return file_volume_server_proto_rawDescGZIP(), []int{55} } type VolumeEcShardsMountRequest struct { @@ -2878,7 +3012,7 @@ type VolumeEcShardsMountRequest struct { func (x *VolumeEcShardsMountRequest) Reset() { *x = VolumeEcShardsMountRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[54] + mi := &file_volume_server_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2891,7 +3025,7 @@ func (x *VolumeEcShardsMountRequest) String() string { func (*VolumeEcShardsMountRequest) ProtoMessage() {} func (x *VolumeEcShardsMountRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[54] + mi := &file_volume_server_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2904,7 +3038,7 @@ func (x *VolumeEcShardsMountRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsMountRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsMountRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{54} + return file_volume_server_proto_rawDescGZIP(), []int{56} } func (x *VolumeEcShardsMountRequest) GetVolumeId() uint32 { @@ -2937,7 +3071,7 @@ type VolumeEcShardsMountResponse struct { func (x *VolumeEcShardsMountResponse) Reset() { *x = VolumeEcShardsMountResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[55] + mi := &file_volume_server_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2950,7 +3084,7 @@ func (x *VolumeEcShardsMountResponse) String() string { func (*VolumeEcShardsMountResponse) ProtoMessage() {} func (x *VolumeEcShardsMountResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[55] + mi := &file_volume_server_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2963,7 +3097,7 @@ func (x *VolumeEcShardsMountResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsMountResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsMountResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{55} + return file_volume_server_proto_rawDescGZIP(), []int{57} } type VolumeEcShardsUnmountRequest struct { @@ -2978,7 +3112,7 @@ type VolumeEcShardsUnmountRequest struct { func (x *VolumeEcShardsUnmountRequest) Reset() { *x = VolumeEcShardsUnmountRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[56] + mi := &file_volume_server_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2991,7 +3125,7 @@ func (x *VolumeEcShardsUnmountRequest) String() string { func (*VolumeEcShardsUnmountRequest) ProtoMessage() {} func (x *VolumeEcShardsUnmountRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[56] + mi := &file_volume_server_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3004,7 +3138,7 @@ func (x *VolumeEcShardsUnmountRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsUnmountRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsUnmountRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{56} + return file_volume_server_proto_rawDescGZIP(), []int{58} } func (x *VolumeEcShardsUnmountRequest) GetVolumeId() uint32 { @@ -3030,7 +3164,7 @@ type VolumeEcShardsUnmountResponse struct { func (x *VolumeEcShardsUnmountResponse) Reset() { *x = VolumeEcShardsUnmountResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[57] + mi := &file_volume_server_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3043,7 +3177,7 @@ func (x *VolumeEcShardsUnmountResponse) String() string { func (*VolumeEcShardsUnmountResponse) ProtoMessage() {} func (x *VolumeEcShardsUnmountResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[57] + mi := &file_volume_server_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3056,7 +3190,7 @@ func (x *VolumeEcShardsUnmountResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsUnmountResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsUnmountResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{57} + return file_volume_server_proto_rawDescGZIP(), []int{59} } type VolumeEcShardReadRequest struct { @@ -3074,7 +3208,7 @@ type VolumeEcShardReadRequest struct { func (x *VolumeEcShardReadRequest) Reset() { *x = VolumeEcShardReadRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[58] + mi := &file_volume_server_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3087,7 +3221,7 @@ func (x *VolumeEcShardReadRequest) String() string { func (*VolumeEcShardReadRequest) ProtoMessage() {} func (x *VolumeEcShardReadRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[58] + mi := &file_volume_server_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3100,7 +3234,7 @@ func (x *VolumeEcShardReadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardReadRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardReadRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{58} + return file_volume_server_proto_rawDescGZIP(), []int{60} } func (x *VolumeEcShardReadRequest) GetVolumeId() uint32 { @@ -3150,7 +3284,7 @@ type VolumeEcShardReadResponse struct { func (x *VolumeEcShardReadResponse) Reset() { *x = VolumeEcShardReadResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[59] + mi := &file_volume_server_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3163,7 +3297,7 @@ func (x *VolumeEcShardReadResponse) String() string { func (*VolumeEcShardReadResponse) ProtoMessage() {} func (x *VolumeEcShardReadResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[59] + mi := &file_volume_server_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3176,7 +3310,7 @@ func (x *VolumeEcShardReadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardReadResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardReadResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{59} + return file_volume_server_proto_rawDescGZIP(), []int{61} } func (x *VolumeEcShardReadResponse) GetData() []byte { @@ -3207,7 +3341,7 @@ type VolumeEcBlobDeleteRequest struct { func (x *VolumeEcBlobDeleteRequest) Reset() { *x = VolumeEcBlobDeleteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[60] + mi := &file_volume_server_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3220,7 +3354,7 @@ func (x *VolumeEcBlobDeleteRequest) String() string { func (*VolumeEcBlobDeleteRequest) ProtoMessage() {} func (x *VolumeEcBlobDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[60] + mi := &file_volume_server_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3233,7 +3367,7 @@ func (x *VolumeEcBlobDeleteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcBlobDeleteRequest.ProtoReflect.Descriptor instead. func (*VolumeEcBlobDeleteRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{60} + return file_volume_server_proto_rawDescGZIP(), []int{62} } func (x *VolumeEcBlobDeleteRequest) GetVolumeId() uint32 { @@ -3273,7 +3407,7 @@ type VolumeEcBlobDeleteResponse struct { func (x *VolumeEcBlobDeleteResponse) Reset() { *x = VolumeEcBlobDeleteResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[61] + mi := &file_volume_server_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3286,7 +3420,7 @@ func (x *VolumeEcBlobDeleteResponse) String() string { func (*VolumeEcBlobDeleteResponse) ProtoMessage() {} func (x *VolumeEcBlobDeleteResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[61] + mi := &file_volume_server_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3299,7 +3433,7 @@ func (x *VolumeEcBlobDeleteResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcBlobDeleteResponse.ProtoReflect.Descriptor instead. func (*VolumeEcBlobDeleteResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{61} + return file_volume_server_proto_rawDescGZIP(), []int{63} } type VolumeEcShardsToVolumeRequest struct { @@ -3314,7 +3448,7 @@ type VolumeEcShardsToVolumeRequest struct { func (x *VolumeEcShardsToVolumeRequest) Reset() { *x = VolumeEcShardsToVolumeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[62] + mi := &file_volume_server_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3327,7 +3461,7 @@ func (x *VolumeEcShardsToVolumeRequest) String() string { func (*VolumeEcShardsToVolumeRequest) ProtoMessage() {} func (x *VolumeEcShardsToVolumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[62] + mi := &file_volume_server_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3340,7 +3474,7 @@ func (x *VolumeEcShardsToVolumeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsToVolumeRequest.ProtoReflect.Descriptor instead. func (*VolumeEcShardsToVolumeRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{62} + return file_volume_server_proto_rawDescGZIP(), []int{64} } func (x *VolumeEcShardsToVolumeRequest) GetVolumeId() uint32 { @@ -3366,7 +3500,7 @@ type VolumeEcShardsToVolumeResponse struct { func (x *VolumeEcShardsToVolumeResponse) Reset() { *x = VolumeEcShardsToVolumeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[63] + mi := &file_volume_server_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3379,7 +3513,7 @@ func (x *VolumeEcShardsToVolumeResponse) String() string { func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} func (x *VolumeEcShardsToVolumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[63] + mi := &file_volume_server_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3392,7 +3526,7 @@ func (x *VolumeEcShardsToVolumeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeEcShardsToVolumeResponse.ProtoReflect.Descriptor instead. func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{63} + return file_volume_server_proto_rawDescGZIP(), []int{65} } type ReadVolumeFileStatusRequest struct { @@ -3406,7 +3540,7 @@ type ReadVolumeFileStatusRequest struct { func (x *ReadVolumeFileStatusRequest) Reset() { *x = ReadVolumeFileStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[64] + mi := &file_volume_server_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3419,7 +3553,7 @@ func (x *ReadVolumeFileStatusRequest) String() string { func (*ReadVolumeFileStatusRequest) ProtoMessage() {} func (x *ReadVolumeFileStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[64] + mi := &file_volume_server_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3432,7 +3566,7 @@ func (x *ReadVolumeFileStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadVolumeFileStatusRequest.ProtoReflect.Descriptor instead. func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{64} + return file_volume_server_proto_rawDescGZIP(), []int{66} } func (x *ReadVolumeFileStatusRequest) GetVolumeId() uint32 { @@ -3461,7 +3595,7 @@ type ReadVolumeFileStatusResponse struct { func (x *ReadVolumeFileStatusResponse) Reset() { *x = ReadVolumeFileStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[65] + mi := &file_volume_server_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3474,7 +3608,7 @@ func (x *ReadVolumeFileStatusResponse) String() string { func (*ReadVolumeFileStatusResponse) ProtoMessage() {} func (x *ReadVolumeFileStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[65] + mi := &file_volume_server_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3487,7 +3621,7 @@ func (x *ReadVolumeFileStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadVolumeFileStatusResponse.ProtoReflect.Descriptor instead. func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{65} + return file_volume_server_proto_rawDescGZIP(), []int{67} } func (x *ReadVolumeFileStatusResponse) GetVolumeId() uint32 { @@ -3570,7 +3704,7 @@ type DiskStatus struct { func (x *DiskStatus) Reset() { *x = DiskStatus{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[66] + mi := &file_volume_server_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3583,7 +3717,7 @@ func (x *DiskStatus) String() string { func (*DiskStatus) ProtoMessage() {} func (x *DiskStatus) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[66] + mi := &file_volume_server_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3596,7 +3730,7 @@ func (x *DiskStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use DiskStatus.ProtoReflect.Descriptor instead. func (*DiskStatus) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{66} + return file_volume_server_proto_rawDescGZIP(), []int{68} } func (x *DiskStatus) GetDir() string { @@ -3665,7 +3799,7 @@ type MemStatus struct { func (x *MemStatus) Reset() { *x = MemStatus{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[67] + mi := &file_volume_server_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3678,7 +3812,7 @@ func (x *MemStatus) String() string { func (*MemStatus) ProtoMessage() {} func (x *MemStatus) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[67] + mi := &file_volume_server_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3691,7 +3825,7 @@ func (x *MemStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use MemStatus.ProtoReflect.Descriptor instead. func (*MemStatus) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{67} + return file_volume_server_proto_rawDescGZIP(), []int{69} } func (x *MemStatus) GetGoroutines() int32 { @@ -3761,7 +3895,7 @@ type RemoteFile struct { func (x *RemoteFile) Reset() { *x = RemoteFile{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[68] + mi := &file_volume_server_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3774,7 +3908,7 @@ func (x *RemoteFile) String() string { func (*RemoteFile) ProtoMessage() {} func (x *RemoteFile) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[68] + mi := &file_volume_server_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3787,7 +3921,7 @@ func (x *RemoteFile) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoteFile.ProtoReflect.Descriptor instead. func (*RemoteFile) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{68} + return file_volume_server_proto_rawDescGZIP(), []int{70} } func (x *RemoteFile) GetBackendType() string { @@ -3852,7 +3986,7 @@ type VolumeInfo struct { func (x *VolumeInfo) Reset() { *x = VolumeInfo{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[69] + mi := &file_volume_server_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3865,7 +3999,7 @@ func (x *VolumeInfo) String() string { func (*VolumeInfo) ProtoMessage() {} func (x *VolumeInfo) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[69] + mi := &file_volume_server_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3878,7 +4012,7 @@ func (x *VolumeInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeInfo.ProtoReflect.Descriptor instead. func (*VolumeInfo) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{69} + return file_volume_server_proto_rawDescGZIP(), []int{71} } func (x *VolumeInfo) GetFiles() []*RemoteFile { @@ -3917,7 +4051,7 @@ type VolumeTierMoveDatToRemoteRequest struct { func (x *VolumeTierMoveDatToRemoteRequest) Reset() { *x = VolumeTierMoveDatToRemoteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[70] + mi := &file_volume_server_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3930,7 +4064,7 @@ func (x *VolumeTierMoveDatToRemoteRequest) String() string { func (*VolumeTierMoveDatToRemoteRequest) ProtoMessage() {} func (x *VolumeTierMoveDatToRemoteRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[70] + mi := &file_volume_server_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3943,7 +4077,7 @@ func (x *VolumeTierMoveDatToRemoteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeTierMoveDatToRemoteRequest.ProtoReflect.Descriptor instead. func (*VolumeTierMoveDatToRemoteRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{70} + return file_volume_server_proto_rawDescGZIP(), []int{72} } func (x *VolumeTierMoveDatToRemoteRequest) GetVolumeId() uint32 { @@ -3986,7 +4120,7 @@ type VolumeTierMoveDatToRemoteResponse struct { func (x *VolumeTierMoveDatToRemoteResponse) Reset() { *x = VolumeTierMoveDatToRemoteResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[71] + mi := &file_volume_server_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3999,7 +4133,7 @@ func (x *VolumeTierMoveDatToRemoteResponse) String() string { func (*VolumeTierMoveDatToRemoteResponse) ProtoMessage() {} func (x *VolumeTierMoveDatToRemoteResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[71] + mi := &file_volume_server_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4012,7 +4146,7 @@ func (x *VolumeTierMoveDatToRemoteResponse) ProtoReflect() protoreflect.Message // Deprecated: Use VolumeTierMoveDatToRemoteResponse.ProtoReflect.Descriptor instead. func (*VolumeTierMoveDatToRemoteResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{71} + return file_volume_server_proto_rawDescGZIP(), []int{73} } func (x *VolumeTierMoveDatToRemoteResponse) GetProcessed() int64 { @@ -4042,7 +4176,7 @@ type VolumeTierMoveDatFromRemoteRequest struct { func (x *VolumeTierMoveDatFromRemoteRequest) Reset() { *x = VolumeTierMoveDatFromRemoteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[72] + mi := &file_volume_server_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4055,7 +4189,7 @@ func (x *VolumeTierMoveDatFromRemoteRequest) String() string { func (*VolumeTierMoveDatFromRemoteRequest) ProtoMessage() {} func (x *VolumeTierMoveDatFromRemoteRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[72] + mi := &file_volume_server_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4068,7 +4202,7 @@ func (x *VolumeTierMoveDatFromRemoteRequest) ProtoReflect() protoreflect.Message // Deprecated: Use VolumeTierMoveDatFromRemoteRequest.ProtoReflect.Descriptor instead. func (*VolumeTierMoveDatFromRemoteRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{72} + return file_volume_server_proto_rawDescGZIP(), []int{74} } func (x *VolumeTierMoveDatFromRemoteRequest) GetVolumeId() uint32 { @@ -4104,7 +4238,7 @@ type VolumeTierMoveDatFromRemoteResponse struct { func (x *VolumeTierMoveDatFromRemoteResponse) Reset() { *x = VolumeTierMoveDatFromRemoteResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[73] + mi := &file_volume_server_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4117,7 +4251,7 @@ func (x *VolumeTierMoveDatFromRemoteResponse) String() string { func (*VolumeTierMoveDatFromRemoteResponse) ProtoMessage() {} func (x *VolumeTierMoveDatFromRemoteResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[73] + mi := &file_volume_server_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4130,7 +4264,7 @@ func (x *VolumeTierMoveDatFromRemoteResponse) ProtoReflect() protoreflect.Messag // Deprecated: Use VolumeTierMoveDatFromRemoteResponse.ProtoReflect.Descriptor instead. func (*VolumeTierMoveDatFromRemoteResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{73} + return file_volume_server_proto_rawDescGZIP(), []int{75} } func (x *VolumeTierMoveDatFromRemoteResponse) GetProcessed() int64 { @@ -4156,7 +4290,7 @@ type VolumeServerStatusRequest struct { func (x *VolumeServerStatusRequest) Reset() { *x = VolumeServerStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[74] + mi := &file_volume_server_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4169,7 +4303,7 @@ func (x *VolumeServerStatusRequest) String() string { func (*VolumeServerStatusRequest) ProtoMessage() {} func (x *VolumeServerStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[74] + mi := &file_volume_server_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4182,7 +4316,7 @@ func (x *VolumeServerStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeServerStatusRequest.ProtoReflect.Descriptor instead. func (*VolumeServerStatusRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{74} + return file_volume_server_proto_rawDescGZIP(), []int{76} } type VolumeServerStatusResponse struct { @@ -4192,12 +4326,15 @@ type VolumeServerStatusResponse struct { DiskStatuses []*DiskStatus `protobuf:"bytes,1,rep,name=disk_statuses,json=diskStatuses,proto3" json:"disk_statuses,omitempty"` MemoryStatus *MemStatus `protobuf:"bytes,2,opt,name=memory_status,json=memoryStatus,proto3" json:"memory_status,omitempty"` + Version string `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + DataCenter string `protobuf:"bytes,4,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` + Rack string `protobuf:"bytes,5,opt,name=rack,proto3" json:"rack,omitempty"` } func (x *VolumeServerStatusResponse) Reset() { *x = VolumeServerStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[75] + mi := &file_volume_server_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4210,7 +4347,7 @@ func (x *VolumeServerStatusResponse) String() string { func (*VolumeServerStatusResponse) ProtoMessage() {} func (x *VolumeServerStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[75] + mi := &file_volume_server_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4223,7 +4360,7 @@ func (x *VolumeServerStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeServerStatusResponse.ProtoReflect.Descriptor instead. func (*VolumeServerStatusResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{75} + return file_volume_server_proto_rawDescGZIP(), []int{77} } func (x *VolumeServerStatusResponse) GetDiskStatuses() []*DiskStatus { @@ -4240,6 +4377,27 @@ func (x *VolumeServerStatusResponse) GetMemoryStatus() *MemStatus { return nil } +func (x *VolumeServerStatusResponse) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *VolumeServerStatusResponse) GetDataCenter() string { + if x != nil { + return x.DataCenter + } + return "" +} + +func (x *VolumeServerStatusResponse) GetRack() string { + if x != nil { + return x.Rack + } + return "" +} + type VolumeServerLeaveRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -4249,7 +4407,7 @@ type VolumeServerLeaveRequest struct { func (x *VolumeServerLeaveRequest) Reset() { *x = VolumeServerLeaveRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[76] + mi := &file_volume_server_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4262,7 +4420,7 @@ func (x *VolumeServerLeaveRequest) String() string { func (*VolumeServerLeaveRequest) ProtoMessage() {} func (x *VolumeServerLeaveRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[76] + mi := &file_volume_server_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4275,7 +4433,7 @@ func (x *VolumeServerLeaveRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeServerLeaveRequest.ProtoReflect.Descriptor instead. func (*VolumeServerLeaveRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{76} + return file_volume_server_proto_rawDescGZIP(), []int{78} } type VolumeServerLeaveResponse struct { @@ -4287,7 +4445,7 @@ type VolumeServerLeaveResponse struct { func (x *VolumeServerLeaveResponse) Reset() { *x = VolumeServerLeaveResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[77] + mi := &file_volume_server_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4300,7 +4458,7 @@ func (x *VolumeServerLeaveResponse) String() string { func (*VolumeServerLeaveResponse) ProtoMessage() {} func (x *VolumeServerLeaveResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[77] + mi := &file_volume_server_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4313,7 +4471,7 @@ func (x *VolumeServerLeaveResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeServerLeaveResponse.ProtoReflect.Descriptor instead. func (*VolumeServerLeaveResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{77} + return file_volume_server_proto_rawDescGZIP(), []int{79} } // remote storage @@ -4322,11 +4480,13 @@ type FetchAndWriteNeedleRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` - NeedleId uint64 `protobuf:"varint,2,opt,name=needle_id,json=needleId,proto3" json:"needle_id,omitempty"` - Cookie uint32 `protobuf:"varint,3,opt,name=cookie,proto3" json:"cookie,omitempty"` - Offset int64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` - Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + NeedleId uint64 `protobuf:"varint,2,opt,name=needle_id,json=needleId,proto3" json:"needle_id,omitempty"` + Cookie uint32 `protobuf:"varint,3,opt,name=cookie,proto3" json:"cookie,omitempty"` + Offset int64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` + Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` + Replicas []*FetchAndWriteNeedleRequest_Replica `protobuf:"bytes,6,rep,name=replicas,proto3" json:"replicas,omitempty"` + Auth string `protobuf:"bytes,7,opt,name=auth,proto3" json:"auth,omitempty"` // remote conf RemoteConf *remote_pb.RemoteConf `protobuf:"bytes,15,opt,name=remote_conf,json=remoteConf,proto3" json:"remote_conf,omitempty"` RemoteLocation *remote_pb.RemoteStorageLocation `protobuf:"bytes,16,opt,name=remote_location,json=remoteLocation,proto3" json:"remote_location,omitempty"` @@ -4335,7 +4495,7 @@ type FetchAndWriteNeedleRequest struct { func (x *FetchAndWriteNeedleRequest) Reset() { *x = FetchAndWriteNeedleRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[78] + mi := &file_volume_server_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4348,7 +4508,7 @@ func (x *FetchAndWriteNeedleRequest) String() string { func (*FetchAndWriteNeedleRequest) ProtoMessage() {} func (x *FetchAndWriteNeedleRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[78] + mi := &file_volume_server_proto_msgTypes[80] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4361,7 +4521,7 @@ func (x *FetchAndWriteNeedleRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchAndWriteNeedleRequest.ProtoReflect.Descriptor instead. func (*FetchAndWriteNeedleRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{78} + return file_volume_server_proto_rawDescGZIP(), []int{80} } func (x *FetchAndWriteNeedleRequest) GetVolumeId() uint32 { @@ -4399,6 +4559,20 @@ func (x *FetchAndWriteNeedleRequest) GetSize() int64 { return 0 } +func (x *FetchAndWriteNeedleRequest) GetReplicas() []*FetchAndWriteNeedleRequest_Replica { + if x != nil { + return x.Replicas + } + return nil +} + +func (x *FetchAndWriteNeedleRequest) GetAuth() string { + if x != nil { + return x.Auth + } + return "" +} + func (x *FetchAndWriteNeedleRequest) GetRemoteConf() *remote_pb.RemoteConf { if x != nil { return x.RemoteConf @@ -4422,7 +4596,7 @@ type FetchAndWriteNeedleResponse struct { func (x *FetchAndWriteNeedleResponse) Reset() { *x = FetchAndWriteNeedleResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[79] + mi := &file_volume_server_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4435,7 +4609,7 @@ func (x *FetchAndWriteNeedleResponse) String() string { func (*FetchAndWriteNeedleResponse) ProtoMessage() {} func (x *FetchAndWriteNeedleResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[79] + mi := &file_volume_server_proto_msgTypes[81] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4448,7 +4622,7 @@ func (x *FetchAndWriteNeedleResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchAndWriteNeedleResponse.ProtoReflect.Descriptor instead. func (*FetchAndWriteNeedleResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{79} + return file_volume_server_proto_rawDescGZIP(), []int{81} } // select on volume servers @@ -4467,7 +4641,7 @@ type QueryRequest struct { func (x *QueryRequest) Reset() { *x = QueryRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[80] + mi := &file_volume_server_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4480,7 +4654,7 @@ func (x *QueryRequest) String() string { func (*QueryRequest) ProtoMessage() {} func (x *QueryRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[80] + mi := &file_volume_server_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4493,7 +4667,7 @@ func (x *QueryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use QueryRequest.ProtoReflect.Descriptor instead. func (*QueryRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80} + return file_volume_server_proto_rawDescGZIP(), []int{82} } func (x *QueryRequest) GetSelections() []string { @@ -4542,7 +4716,7 @@ type QueriedStripe struct { func (x *QueriedStripe) Reset() { *x = QueriedStripe{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[81] + mi := &file_volume_server_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4555,7 +4729,7 @@ func (x *QueriedStripe) String() string { func (*QueriedStripe) ProtoMessage() {} func (x *QueriedStripe) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[81] + mi := &file_volume_server_proto_msgTypes[83] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4568,7 +4742,7 @@ func (x *QueriedStripe) ProtoReflect() protoreflect.Message { // Deprecated: Use QueriedStripe.ProtoReflect.Descriptor instead. func (*QueriedStripe) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{81} + return file_volume_server_proto_rawDescGZIP(), []int{83} } func (x *QueriedStripe) GetRecords() []byte { @@ -4590,7 +4764,7 @@ type VolumeNeedleStatusRequest struct { func (x *VolumeNeedleStatusRequest) Reset() { *x = VolumeNeedleStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[82] + mi := &file_volume_server_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4603,7 +4777,7 @@ func (x *VolumeNeedleStatusRequest) String() string { func (*VolumeNeedleStatusRequest) ProtoMessage() {} func (x *VolumeNeedleStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[82] + mi := &file_volume_server_proto_msgTypes[84] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4616,7 +4790,7 @@ func (x *VolumeNeedleStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeNeedleStatusRequest.ProtoReflect.Descriptor instead. func (*VolumeNeedleStatusRequest) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{82} + return file_volume_server_proto_rawDescGZIP(), []int{84} } func (x *VolumeNeedleStatusRequest) GetVolumeId() uint32 { @@ -4649,7 +4823,7 @@ type VolumeNeedleStatusResponse struct { func (x *VolumeNeedleStatusResponse) Reset() { *x = VolumeNeedleStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[83] + mi := &file_volume_server_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4662,7 +4836,7 @@ func (x *VolumeNeedleStatusResponse) String() string { func (*VolumeNeedleStatusResponse) ProtoMessage() {} func (x *VolumeNeedleStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[83] + mi := &file_volume_server_proto_msgTypes[85] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4675,7 +4849,7 @@ func (x *VolumeNeedleStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeNeedleStatusResponse.ProtoReflect.Descriptor instead. func (*VolumeNeedleStatusResponse) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{83} + return file_volume_server_proto_rawDescGZIP(), []int{85} } func (x *VolumeNeedleStatusResponse) GetNeedleId() uint64 { @@ -4720,33 +4894,32 @@ func (x *VolumeNeedleStatusResponse) GetTtl() string { return "" } -type QueryRequest_Filter struct { +type PingRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"` - Operand string `protobuf:"bytes,2,opt,name=operand,proto3" json:"operand,omitempty"` - Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // default to ping itself + TargetType string `protobuf:"bytes,2,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` } -func (x *QueryRequest_Filter) Reset() { - *x = QueryRequest_Filter{} +func (x *PingRequest) Reset() { + *x = PingRequest{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[84] + mi := &file_volume_server_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *QueryRequest_Filter) String() string { +func (x *PingRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryRequest_Filter) ProtoMessage() {} +func (*PingRequest) ProtoMessage() {} -func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[84] +func (x *PingRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[86] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4757,61 +4930,52 @@ func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryRequest_Filter.ProtoReflect.Descriptor instead. -func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 0} -} - -func (x *QueryRequest_Filter) GetField() string { - if x != nil { - return x.Field - } - return "" +// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. +func (*PingRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{86} } -func (x *QueryRequest_Filter) GetOperand() string { +func (x *PingRequest) GetTarget() string { if x != nil { - return x.Operand + return x.Target } return "" } -func (x *QueryRequest_Filter) GetValue() string { +func (x *PingRequest) GetTargetType() string { if x != nil { - return x.Value + return x.TargetType } return "" } -type QueryRequest_InputSerialization struct { +type PingResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // NONE | GZIP | BZIP2 - CompressionType string `protobuf:"bytes,1,opt,name=compression_type,json=compressionType,proto3" json:"compression_type,omitempty"` - CsvInput *QueryRequest_InputSerialization_CSVInput `protobuf:"bytes,2,opt,name=csv_input,json=csvInput,proto3" json:"csv_input,omitempty"` - JsonInput *QueryRequest_InputSerialization_JSONInput `protobuf:"bytes,3,opt,name=json_input,json=jsonInput,proto3" json:"json_input,omitempty"` - ParquetInput *QueryRequest_InputSerialization_ParquetInput `protobuf:"bytes,4,opt,name=parquet_input,json=parquetInput,proto3" json:"parquet_input,omitempty"` + StartTimeNs int64 `protobuf:"varint,1,opt,name=start_time_ns,json=startTimeNs,proto3" json:"start_time_ns,omitempty"` + RemoteTimeNs int64 `protobuf:"varint,2,opt,name=remote_time_ns,json=remoteTimeNs,proto3" json:"remote_time_ns,omitempty"` + StopTimeNs int64 `protobuf:"varint,3,opt,name=stop_time_ns,json=stopTimeNs,proto3" json:"stop_time_ns,omitempty"` } -func (x *QueryRequest_InputSerialization) Reset() { - *x = QueryRequest_InputSerialization{} +func (x *PingResponse) Reset() { + *x = PingResponse{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[85] + mi := &file_volume_server_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *QueryRequest_InputSerialization) String() string { +func (x *PingResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryRequest_InputSerialization) ProtoMessage() {} +func (*PingResponse) ProtoMessage() {} -func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[85] +func (x *PingResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[87] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4822,65 +4986,59 @@ func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryRequest_InputSerialization.ProtoReflect.Descriptor instead. -func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 1} -} - -func (x *QueryRequest_InputSerialization) GetCompressionType() string { - if x != nil { - return x.CompressionType - } - return "" +// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. +func (*PingResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{87} } -func (x *QueryRequest_InputSerialization) GetCsvInput() *QueryRequest_InputSerialization_CSVInput { +func (x *PingResponse) GetStartTimeNs() int64 { if x != nil { - return x.CsvInput + return x.StartTimeNs } - return nil + return 0 } -func (x *QueryRequest_InputSerialization) GetJsonInput() *QueryRequest_InputSerialization_JSONInput { +func (x *PingResponse) GetRemoteTimeNs() int64 { if x != nil { - return x.JsonInput + return x.RemoteTimeNs } - return nil + return 0 } -func (x *QueryRequest_InputSerialization) GetParquetInput() *QueryRequest_InputSerialization_ParquetInput { +func (x *PingResponse) GetStopTimeNs() int64 { if x != nil { - return x.ParquetInput + return x.StopTimeNs } - return nil + return 0 } -type QueryRequest_OutputSerialization struct { +type FetchAndWriteNeedleRequest_Replica struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - CsvOutput *QueryRequest_OutputSerialization_CSVOutput `protobuf:"bytes,2,opt,name=csv_output,json=csvOutput,proto3" json:"csv_output,omitempty"` - JsonOutput *QueryRequest_OutputSerialization_JSONOutput `protobuf:"bytes,3,opt,name=json_output,json=jsonOutput,proto3" json:"json_output,omitempty"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl,proto3" json:"public_url,omitempty"` + GrpcPort int32 `protobuf:"varint,3,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` } -func (x *QueryRequest_OutputSerialization) Reset() { - *x = QueryRequest_OutputSerialization{} +func (x *FetchAndWriteNeedleRequest_Replica) Reset() { + *x = FetchAndWriteNeedleRequest_Replica{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[86] + mi := &file_volume_server_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *QueryRequest_OutputSerialization) String() string { +func (x *FetchAndWriteNeedleRequest_Replica) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryRequest_OutputSerialization) ProtoMessage() {} +func (*FetchAndWriteNeedleRequest_Replica) ProtoMessage() {} -func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[86] +func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[88] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4891,57 +5049,59 @@ func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryRequest_OutputSerialization.ProtoReflect.Descriptor instead. -func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 2} +// Deprecated: Use FetchAndWriteNeedleRequest_Replica.ProtoReflect.Descriptor instead. +func (*FetchAndWriteNeedleRequest_Replica) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{80, 0} } -func (x *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput { +func (x *FetchAndWriteNeedleRequest_Replica) GetUrl() string { if x != nil { - return x.CsvOutput + return x.Url } - return nil + return "" } -func (x *QueryRequest_OutputSerialization) GetJsonOutput() *QueryRequest_OutputSerialization_JSONOutput { +func (x *FetchAndWriteNeedleRequest_Replica) GetPublicUrl() string { if x != nil { - return x.JsonOutput + return x.PublicUrl } - return nil + return "" } -type QueryRequest_InputSerialization_CSVInput struct { +func (x *FetchAndWriteNeedleRequest_Replica) GetGrpcPort() int32 { + if x != nil { + return x.GrpcPort + } + return 0 +} + +type QueryRequest_Filter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - FileHeaderInfo string `protobuf:"bytes,1,opt,name=file_header_info,json=fileHeaderInfo,proto3" json:"file_header_info,omitempty"` // Valid values: NONE | USE | IGNORE - RecordDelimiter string `protobuf:"bytes,2,opt,name=record_delimiter,json=recordDelimiter,proto3" json:"record_delimiter,omitempty"` // Default: \n - FieldDelimiter string `protobuf:"bytes,3,opt,name=field_delimiter,json=fieldDelimiter,proto3" json:"field_delimiter,omitempty"` // Default: , - QuoteCharactoer string `protobuf:"bytes,4,opt,name=quote_charactoer,json=quoteCharactoer,proto3" json:"quote_charactoer,omitempty"` // Default: " - QuoteEscapeCharacter string `protobuf:"bytes,5,opt,name=quote_escape_character,json=quoteEscapeCharacter,proto3" json:"quote_escape_character,omitempty"` // Default: " - Comments string `protobuf:"bytes,6,opt,name=comments,proto3" json:"comments,omitempty"` // Default: # - // If true, records might contain record delimiters within quote characters - AllowQuotedRecordDelimiter bool `protobuf:"varint,7,opt,name=allow_quoted_record_delimiter,json=allowQuotedRecordDelimiter,proto3" json:"allow_quoted_record_delimiter,omitempty"` // default False. + Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"` + Operand string `protobuf:"bytes,2,opt,name=operand,proto3" json:"operand,omitempty"` + Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` } -func (x *QueryRequest_InputSerialization_CSVInput) Reset() { - *x = QueryRequest_InputSerialization_CSVInput{} +func (x *QueryRequest_Filter) Reset() { + *x = QueryRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[87] + mi := &file_volume_server_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *QueryRequest_InputSerialization_CSVInput) String() string { +func (x *QueryRequest_Filter) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} +func (*QueryRequest_Filter) ProtoMessage() {} -func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[87] +func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[89] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4952,12 +5112,207 @@ func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.M return mi.MessageOf(x) } -// Deprecated: Use QueryRequest_InputSerialization_CSVInput.ProtoReflect.Descriptor instead. -func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 1, 0} +// Deprecated: Use QueryRequest_Filter.ProtoReflect.Descriptor instead. +func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{82, 0} } -func (x *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string { +func (x *QueryRequest_Filter) GetField() string { + if x != nil { + return x.Field + } + return "" +} + +func (x *QueryRequest_Filter) GetOperand() string { + if x != nil { + return x.Operand + } + return "" +} + +func (x *QueryRequest_Filter) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type QueryRequest_InputSerialization struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // NONE | GZIP | BZIP2 + CompressionType string `protobuf:"bytes,1,opt,name=compression_type,json=compressionType,proto3" json:"compression_type,omitempty"` + CsvInput *QueryRequest_InputSerialization_CSVInput `protobuf:"bytes,2,opt,name=csv_input,json=csvInput,proto3" json:"csv_input,omitempty"` + JsonInput *QueryRequest_InputSerialization_JSONInput `protobuf:"bytes,3,opt,name=json_input,json=jsonInput,proto3" json:"json_input,omitempty"` + ParquetInput *QueryRequest_InputSerialization_ParquetInput `protobuf:"bytes,4,opt,name=parquet_input,json=parquetInput,proto3" json:"parquet_input,omitempty"` +} + +func (x *QueryRequest_InputSerialization) Reset() { + *x = QueryRequest_InputSerialization{} + if protoimpl.UnsafeEnabled { + mi := &file_volume_server_proto_msgTypes[90] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryRequest_InputSerialization) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryRequest_InputSerialization) ProtoMessage() {} + +func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[90] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryRequest_InputSerialization.ProtoReflect.Descriptor instead. +func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{82, 1} +} + +func (x *QueryRequest_InputSerialization) GetCompressionType() string { + if x != nil { + return x.CompressionType + } + return "" +} + +func (x *QueryRequest_InputSerialization) GetCsvInput() *QueryRequest_InputSerialization_CSVInput { + if x != nil { + return x.CsvInput + } + return nil +} + +func (x *QueryRequest_InputSerialization) GetJsonInput() *QueryRequest_InputSerialization_JSONInput { + if x != nil { + return x.JsonInput + } + return nil +} + +func (x *QueryRequest_InputSerialization) GetParquetInput() *QueryRequest_InputSerialization_ParquetInput { + if x != nil { + return x.ParquetInput + } + return nil +} + +type QueryRequest_OutputSerialization struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CsvOutput *QueryRequest_OutputSerialization_CSVOutput `protobuf:"bytes,2,opt,name=csv_output,json=csvOutput,proto3" json:"csv_output,omitempty"` + JsonOutput *QueryRequest_OutputSerialization_JSONOutput `protobuf:"bytes,3,opt,name=json_output,json=jsonOutput,proto3" json:"json_output,omitempty"` +} + +func (x *QueryRequest_OutputSerialization) Reset() { + *x = QueryRequest_OutputSerialization{} + if protoimpl.UnsafeEnabled { + mi := &file_volume_server_proto_msgTypes[91] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryRequest_OutputSerialization) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryRequest_OutputSerialization) ProtoMessage() {} + +func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[91] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryRequest_OutputSerialization.ProtoReflect.Descriptor instead. +func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{82, 2} +} + +func (x *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput { + if x != nil { + return x.CsvOutput + } + return nil +} + +func (x *QueryRequest_OutputSerialization) GetJsonOutput() *QueryRequest_OutputSerialization_JSONOutput { + if x != nil { + return x.JsonOutput + } + return nil +} + +type QueryRequest_InputSerialization_CSVInput struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FileHeaderInfo string `protobuf:"bytes,1,opt,name=file_header_info,json=fileHeaderInfo,proto3" json:"file_header_info,omitempty"` // Valid values: NONE | USE | IGNORE + RecordDelimiter string `protobuf:"bytes,2,opt,name=record_delimiter,json=recordDelimiter,proto3" json:"record_delimiter,omitempty"` // Default: \n + FieldDelimiter string `protobuf:"bytes,3,opt,name=field_delimiter,json=fieldDelimiter,proto3" json:"field_delimiter,omitempty"` // Default: , + QuoteCharactoer string `protobuf:"bytes,4,opt,name=quote_charactoer,json=quoteCharactoer,proto3" json:"quote_charactoer,omitempty"` // Default: " + QuoteEscapeCharacter string `protobuf:"bytes,5,opt,name=quote_escape_character,json=quoteEscapeCharacter,proto3" json:"quote_escape_character,omitempty"` // Default: " + Comments string `protobuf:"bytes,6,opt,name=comments,proto3" json:"comments,omitempty"` // Default: # + // If true, records might contain record delimiters within quote characters + AllowQuotedRecordDelimiter bool `protobuf:"varint,7,opt,name=allow_quoted_record_delimiter,json=allowQuotedRecordDelimiter,proto3" json:"allow_quoted_record_delimiter,omitempty"` // default False. +} + +func (x *QueryRequest_InputSerialization_CSVInput) Reset() { + *x = QueryRequest_InputSerialization_CSVInput{} + if protoimpl.UnsafeEnabled { + mi := &file_volume_server_proto_msgTypes[92] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryRequest_InputSerialization_CSVInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} + +func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[92] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryRequest_InputSerialization_CSVInput.ProtoReflect.Descriptor instead. +func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{82, 1, 0} +} + +func (x *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string { if x != nil { return x.FileHeaderInfo } @@ -5017,7 +5372,7 @@ type QueryRequest_InputSerialization_JSONInput struct { func (x *QueryRequest_InputSerialization_JSONInput) Reset() { *x = QueryRequest_InputSerialization_JSONInput{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[88] + mi := &file_volume_server_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5030,7 +5385,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) String() string { func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[88] + mi := &file_volume_server_proto_msgTypes[93] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5043,7 +5398,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect. // Deprecated: Use QueryRequest_InputSerialization_JSONInput.ProtoReflect.Descriptor instead. func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 1, 1} + return file_volume_server_proto_rawDescGZIP(), []int{82, 1, 1} } func (x *QueryRequest_InputSerialization_JSONInput) GetType() string { @@ -5062,7 +5417,7 @@ type QueryRequest_InputSerialization_ParquetInput struct { func (x *QueryRequest_InputSerialization_ParquetInput) Reset() { *x = QueryRequest_InputSerialization_ParquetInput{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[89] + mi := &file_volume_server_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5075,7 +5430,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) String() string { func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[89] + mi := &file_volume_server_proto_msgTypes[94] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5088,7 +5443,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protorefle // Deprecated: Use QueryRequest_InputSerialization_ParquetInput.ProtoReflect.Descriptor instead. func (*QueryRequest_InputSerialization_ParquetInput) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 1, 2} + return file_volume_server_proto_rawDescGZIP(), []int{82, 1, 2} } type QueryRequest_OutputSerialization_CSVOutput struct { @@ -5106,7 +5461,7 @@ type QueryRequest_OutputSerialization_CSVOutput struct { func (x *QueryRequest_OutputSerialization_CSVOutput) Reset() { *x = QueryRequest_OutputSerialization_CSVOutput{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[90] + mi := &file_volume_server_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5119,7 +5474,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) String() string { func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[90] + mi := &file_volume_server_proto_msgTypes[95] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5132,7 +5487,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect // Deprecated: Use QueryRequest_OutputSerialization_CSVOutput.ProtoReflect.Descriptor instead. func (*QueryRequest_OutputSerialization_CSVOutput) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 2, 0} + return file_volume_server_proto_rawDescGZIP(), []int{82, 2, 0} } func (x *QueryRequest_OutputSerialization_CSVOutput) GetQuoteFields() string { @@ -5181,7 +5536,7 @@ type QueryRequest_OutputSerialization_JSONOutput struct { func (x *QueryRequest_OutputSerialization_JSONOutput) Reset() { *x = QueryRequest_OutputSerialization_JSONOutput{} if protoimpl.UnsafeEnabled { - mi := &file_volume_server_proto_msgTypes[91] + mi := &file_volume_server_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5194,7 +5549,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) String() string { func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[91] + mi := &file_volume_server_proto_msgTypes[96] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5207,7 +5562,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflec // Deprecated: Use QueryRequest_OutputSerialization_JSONOutput.ProtoReflect.Descriptor instead. func (*QueryRequest_OutputSerialization_JSONOutput) Descriptor() ([]byte, []int) { - return file_volume_server_proto_rawDescGZIP(), []int{80, 2, 1} + return file_volume_server_proto_rawDescGZIP(), []int{82, 2, 1} } func (x *QueryRequest_OutputSerialization_JSONOutput) GetRecordDelimiter() string { @@ -5256,571 +5611,620 @@ var file_volume_server_proto_rawDesc = []byte{ 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, - 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x22, 0x1d, 0x0a, 0x1b, 0x56, 0x61, + 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x22, 0x46, 0x0a, 0x1b, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x38, 0x0a, 0x19, 0x56, 0x61, 0x63, - 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0e, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x42, 0x79, 0x74, + 0x65, 0x73, 0x22, 0x38, 0x0a, 0x19, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x1a, + 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0c, 0x69, 0x73, + 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x69, 0x73, 0x52, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x39, 0x0a, 0x1a, + 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, + 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x56, 0x61, 0x63, 0x75, 0x75, + 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x39, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x1a, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xfb, 0x01, + 0x0a, 0x15, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, + 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x61, 0x6c, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x32, 0x0a, 0x16, 0x6d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x6d, 0x65, 0x6d, 0x6f, + 0x72, 0x79, 0x4d, 0x61, 0x70, 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x12, 0x1b, + 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, + 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, + 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0xfb, 0x01, + 0x0a, 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x74, + 0x61, 0x69, 0x6c, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0a, 0x74, 0x61, 0x69, 0x6c, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, + 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x52, + 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x69, 0x64, 0x78, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, + 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x56, 0x0a, 0x1c, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, + 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, + 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, + 0x65, 0x4e, 0x73, 0x22, 0x42, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, + 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x31, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, + 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x15, 0x0a, 0x13, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x33, 0x0a, 0x14, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x17, 0x0a, 0x15, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x32, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x1a, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x49, 0x64, 0x22, 0x16, 0x0a, 0x14, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x38, 0x0a, 0x19, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, + 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x38, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, + 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1c, 0x0a, + 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x57, 0x0a, 0x16, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x49, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2f, 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x32, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x38, 0x0a, 0x14, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x52, 0x65, 0x61, 0x64, 0x4f, - 0x6e, 0x6c, 0x79, 0x22, 0x39, 0x0a, 0x1a, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x6e, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, + 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x44, 0x61, 0x74, 0x61, + 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, + 0x65, 0x22, 0x68, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, + 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x41, 0x74, + 0x4e, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, + 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x94, 0x02, 0x0a, 0x0f, + 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x78, 0x74, 0x12, 0x2f, + 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x76, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x63, 0x6f, 0x6d, + 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x20, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x12, 0x3e, 0x0a, 0x1c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x5f, 0x66, 0x6f, 0x75, + 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, + 0x6e, 0x64, 0x22, 0x5b, 0x0a, 0x10, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x66, 0x69, + 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x6f, 0x64, + 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0c, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x73, 0x4e, 0x73, 0x22, + 0x7d, 0x0a, 0x15, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, + 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x39, + 0x0a, 0x16, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x65, 0x64, + 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x22, 0x87, 0x01, 0x0a, 0x16, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, + 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, + 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, + 0x6c, 0x6f, 0x62, 0x22, 0x19, 0x0a, 0x17, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, + 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, + 0x0a, 0x15, 0x52, 0x65, 0x61, 0x64, 0x41, 0x6c, 0x6c, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x09, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x61, 0x64, 0x41, + 0x6c, 0x6c, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6f, + 0x6b, 0x69, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6c, + 0x6f, 0x62, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, + 0x42, 0x6c, 0x6f, 0x62, 0x22, 0x83, 0x01, 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, + 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, + 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x64, 0x6c, 0x65, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x69, 0x64, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x18, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x65, 0x64, 0x6c, + 0x65, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, + 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, + 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x22, 0x0a, + 0x0d, 0x69, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x73, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x22, 0xb7, 0x01, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, + 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, + 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, + 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x64, 0x6c, 0x65, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x69, 0x64, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x22, 0x1c, 0x0a, 0x1a, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5c, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x1e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5b, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x4b, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x72, 0x65, 0x62, 0x75, 0x69, + 0x6c, 0x74, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x0f, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x49, 0x64, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1d, - 0x0a, 0x1b, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x39, 0x0a, - 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1a, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xfb, 0x01, 0x0a, 0x15, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, - 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, + 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x63, + 0x6f, 0x70, 0x79, 0x5f, 0x65, 0x63, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x45, 0x63, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x12, + 0x28, 0x0a, 0x10, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, + 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x6f, 0x70, + 0x79, 0x5f, 0x65, 0x63, 0x6a, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x45, 0x63, 0x6a, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x22, 0x0a, + 0x0d, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x76, 0x69, 0x66, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x56, 0x69, 0x66, 0x46, 0x69, 0x6c, + 0x65, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x77, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x70, - 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0b, 0x70, 0x72, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x12, 0x20, 0x0a, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, - 0x6c, 0x12, 0x32, 0x0a, 0x16, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x5f, - 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4d, 0x61, 0x70, 0x4d, 0x61, 0x78, 0x53, - 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, 0x0a, 0x17, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x22, 0x1e, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x76, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, + 0x22, 0x1d, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x58, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, + 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x22, 0x1f, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x99, 0x01, 0x0a, 0x18, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x49, 0x64, 0x22, 0xfb, 0x01, 0x0a, 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, - 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, + 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, + 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x22, 0x4e, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x8d, 0x01, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5c, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x1e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3a, 0x0a, 0x1b, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, + 0x22, 0x8a, 0x03, 0x0a, 0x1c, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x3b, + 0x0a, 0x1a, 0x69, 0x64, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x17, 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x69, + 0x64, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0b, 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x3b, 0x0a, 0x1a, 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x17, 0x64, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x22, 0x0a, 0x0d, + 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, + 0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x63, 0x6f, + 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xbb, 0x01, + 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, + 0x64, 0x69, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x69, 0x72, 0x12, 0x10, + 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x61, 0x6c, 0x6c, + 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, + 0x75, 0x73, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x72, 0x63, + 0x65, 0x6e, 0x74, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0b, + 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x46, 0x72, 0x65, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, + 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x02, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x55, 0x73, 0x65, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xa3, 0x01, 0x0a, 0x09, + 0x4d, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x67, 0x6f, 0x72, + 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x67, + 0x6f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x75, + 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x75, 0x73, 0x65, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, + 0x72, 0x65, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, 0x6c, 0x66, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x04, 0x73, 0x65, 0x6c, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x70, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x68, 0x65, 0x61, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x73, + 0x74, 0x61, 0x63, 0x6b, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x73, 0x74, 0x61, 0x63, + 0x6b, 0x22, 0xd8, 0x01, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, + 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, + 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x6f, 0x64, + 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0c, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1c, + 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x7c, 0x0a, 0x0a, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x32, 0x0a, 0x05, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc8, 0x01, 0x0a, 0x20, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, + 0x54, 0x6f, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x18, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x13, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x10, 0x6b, 0x65, 0x65, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x44, 0x61, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x73, 0x0a, 0x21, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, + 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x54, 0x6f, 0x52, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x70, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, + 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x22, 0x92, 0x01, 0x0a, 0x22, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, + 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, - 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, - 0x74, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x69, 0x6c, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, - 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x74, 0x61, 0x69, 0x6c, 0x4f, 0x66, 0x66, - 0x73, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x72, - 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, - 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, - 0x0a, 0x0d, 0x69, 0x64, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, - 0x7a, 0x65, 0x22, 0x56, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x22, 0x42, 0x0a, 0x1d, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, - 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x66, - 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x31, - 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, - 0x64, 0x22, 0x15, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x33, 0x0a, 0x14, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x17, 0x0a, - 0x15, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x32, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x16, 0x0a, 0x14, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x38, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, - 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1c, 0x0a, 0x1a, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, - 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x38, 0x0a, 0x19, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x49, 0x64, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, - 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x57, 0x0a, 0x16, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2f, 0x0a, 0x17, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x32, 0x0a, 0x13, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, - 0x22, 0x38, 0x0a, 0x14, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x72, - 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x69, 0x73, 0x52, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0xcb, 0x01, 0x0a, 0x11, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, - 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, - 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x3f, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, - 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x5f, 0x61, 0x74, - 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x41, - 0x70, 0x70, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x4e, 0x73, 0x22, 0x94, 0x02, 0x0a, 0x0f, 0x43, 0x6f, - 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x78, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x78, 0x74, 0x12, 0x2f, 0x0a, 0x13, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x76, 0x69, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x63, 0x6f, 0x6d, 0x70, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, - 0x0b, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, - 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x65, 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x12, 0x3e, 0x0a, 0x1c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x5f, 0x66, 0x6f, 0x75, 0x6e, 0x64, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, 0x6e, 0x64, - 0x22, 0x35, 0x0a, 0x10, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x7d, 0x0a, 0x15, 0x52, 0x65, 0x61, 0x64, 0x4e, - 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, - 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x39, 0x0a, 0x16, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, - 0x62, 0x22, 0x87, 0x01, 0x0a, 0x16, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, - 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x65, - 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x22, 0x19, 0x0a, 0x17, 0x57, - 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x83, 0x01, 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x64, - 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x69, 0x64, 0x6c, 0x65, 0x54, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x84, 0x01, 0x0a, - 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, - 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x65, - 0x64, 0x6c, 0x65, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0c, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1f, - 0x0a, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x22, 0x0a, 0x0d, 0x69, 0x73, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x73, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x68, - 0x75, 0x6e, 0x6b, 0x22, 0xb7, 0x01, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, - 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, - 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x69, 0x64, 0x6c, - 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x69, 0x64, 0x6c, 0x65, 0x54, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x22, 0x1c, 0x0a, - 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, - 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5c, 0x0a, 0x1d, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x1e, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5b, 0x0a, 0x1c, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x4b, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x72, 0x65, 0x62, - 0x75, 0x69, 0x6c, 0x74, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0f, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x64, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, - 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x12, 0x22, 0x0a, - 0x0d, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x65, 0x63, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x45, 0x63, 0x78, 0x46, 0x69, 0x6c, - 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x44, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x63, - 0x6f, 0x70, 0x79, 0x5f, 0x65, 0x63, 0x6a, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x45, 0x63, 0x6a, 0x46, 0x69, 0x6c, 0x65, 0x12, - 0x22, 0x0a, 0x0d, 0x63, 0x6f, 0x70, 0x79, 0x5f, 0x76, 0x69, 0x66, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6f, 0x70, 0x79, 0x56, 0x69, 0x66, 0x46, - 0x69, 0x6c, 0x65, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x77, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, - 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, - 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x22, 0x1e, 0x0a, 0x1c, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x76, 0x0a, 0x1a, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, - 0x64, 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x58, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0d, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x73, 0x22, 0x1f, 0x0a, 0x1d, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x99, 0x01, 0x0a, - 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, - 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, - 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x0a, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x22, 0x4e, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, - 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, - 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x8d, 0x01, 0x0a, 0x19, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x18, - 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x1c, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5c, 0x0a, 0x1d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x1e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3a, 0x0a, 0x1b, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x49, 0x64, 0x22, 0x8a, 0x03, 0x0a, 0x1c, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, - 0x12, 0x3b, 0x0a, 0x1a, 0x69, 0x64, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x22, 0x0a, - 0x0d, 0x69, 0x64, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x69, 0x64, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, - 0x65, 0x12, 0x3b, 0x0a, 0x1a, 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x64, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x22, - 0x0a, 0x0d, 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x69, - 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, - 0xbb, 0x01, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, - 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x69, 0x72, - 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x61, - 0x6c, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x04, 0x75, 0x73, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, - 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, - 0x52, 0x0b, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x46, 0x72, 0x65, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x55, 0x73, 0x65, 0x64, - 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xa3, 0x01, - 0x0a, 0x09, 0x4d, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x67, - 0x6f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x0a, 0x67, 0x6f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x61, - 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x12, 0x0a, - 0x04, 0x75, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x75, 0x73, 0x65, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, 0x6c, 0x66, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x65, 0x6c, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, - 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x68, 0x65, 0x61, 0x70, 0x12, 0x14, 0x0a, - 0x05, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x73, 0x74, - 0x61, 0x63, 0x6b, 0x22, 0xd8, 0x01, 0x0a, 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, - 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x61, 0x63, 0x6b, 0x65, - 0x6e, 0x64, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, - 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6d, - 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0c, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x7c, - 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x32, 0x0a, 0x05, - 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc8, 0x01, 0x0a, - 0x20, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, - 0x61, 0x74, 0x54, 0x6f, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1e, - 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x38, - 0x0a, 0x18, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, - 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x16, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, - 0x6b, 0x65, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x13, 0x6b, 0x65, 0x65, 0x70, - 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x6b, 0x65, 0x65, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x44, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x73, 0x0a, 0x21, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x54, 0x6f, 0x52, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, - 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x70, 0x72, - 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, - 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x22, 0x92, 0x01, 0x0a, - 0x22, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, - 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, - 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x2f, 0x0a, 0x14, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, - 0x64, 0x61, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, - 0x6b, 0x65, 0x65, 0x70, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x44, 0x61, 0x74, 0x46, 0x69, 0x6c, - 0x65, 0x22, 0x75, 0x0a, 0x23, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, - 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x50, 0x65, - 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x22, 0x1b, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xa1, 0x01, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0d, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, - 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x64, 0x69, 0x73, 0x6b, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x12, 0x40, 0x0a, 0x0d, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, + 0x0a, 0x14, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x64, 0x61, + 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x6b, 0x65, + 0x65, 0x70, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x44, 0x61, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x22, + 0x75, 0x0a, 0x23, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, + 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x65, + 0x73, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, + 0x64, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x02, 0x52, 0x13, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x50, 0x65, 0x72, 0x63, + 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x22, 0x1b, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x22, 0xf0, 0x01, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0d, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, + 0x6b, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x64, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x65, 0x73, 0x12, 0x40, 0x0a, 0x0d, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x4d, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x22, 0x1a, 0x0a, 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0x1b, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0xdc, 0x03, 0x0a, 0x1a, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, + 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, + 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, + 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x50, 0x0a, 0x08, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x4d, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x6d, 0x65, 0x6d, - 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x1b, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x9d, 0x02, 0x0a, 0x1a, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, - 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, - 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6f, - 0x6b, 0x69, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, - 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, - 0x36, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x18, 0x0f, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x0a, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x49, 0x0a, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x1d, 0x0a, 0x1b, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0xf8, 0x0c, 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x72, 0x6f, 0x6d, 0x46, - 0x69, 0x6c, 0x65, 0x49, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x62, 0x0a, 0x13, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x73, - 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, + 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x12, 0x12, + 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, + 0x74, 0x68, 0x12, 0x36, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x0a, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x49, 0x0a, 0x0f, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, 0x07, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, + 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x1d, + 0x0a, 0x1b, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xf8, 0x0c, + 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, + 0x0a, 0x0a, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, + 0x0a, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x72, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x65, 0x49, + 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x12, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, - 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x65, 0x0a, 0x14, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x1a, 0x4e, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x1a, 0xd5, 0x05, 0x0a, 0x12, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x72, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x57, 0x0a, 0x09, 0x63, 0x73, 0x76, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x53, 0x56, 0x49, 0x6e, 0x70, 0x75, - 0x74, 0x52, 0x08, 0x63, 0x73, 0x76, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x5a, 0x0a, 0x0a, 0x6a, - 0x73, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x09, 0x6a, 0x73, - 0x6f, 0x6e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x63, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x71, 0x75, - 0x65, 0x74, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, + 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x12, 0x62, 0x0a, 0x13, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0c, - 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0xc8, 0x02, 0x0a, - 0x08, 0x43, 0x53, 0x56, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, - 0x65, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, - 0x0a, 0x0f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x71, 0x75, 0x6f, 0x74, 0x65, - 0x5f, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, - 0x65, 0x72, 0x12, 0x34, 0x0a, 0x16, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x73, 0x63, 0x61, - 0x70, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x14, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x43, - 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x41, 0x0a, 0x1d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x71, 0x75, - 0x6f, 0x74, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x61, 0x6c, 0x6c, - 0x6f, 0x77, 0x51, 0x75, 0x6f, 0x74, 0x65, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, 0x65, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x1a, 0x1f, 0x0a, 0x09, 0x4a, 0x53, 0x4f, 0x4e, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x1a, 0x0e, 0x0a, 0x0c, 0x50, 0x61, 0x72, 0x71, - 0x75, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0xf1, 0x03, 0x0a, 0x13, 0x4f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x5b, 0x0a, 0x0a, 0x63, 0x73, 0x76, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x53, 0x56, 0x4f, 0x75, 0x74, 0x70, - 0x75, 0x74, 0x52, 0x09, 0x63, 0x73, 0x76, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x5e, 0x0a, - 0x0b, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, - 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x4f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x52, 0x0a, 0x6a, 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x1a, 0xe3, 0x01, - 0x0a, 0x09, 0x43, 0x53, 0x56, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x71, - 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x29, - 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x72, - 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x71, 0x75, - 0x6f, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x12, 0x34, 0x0a, - 0x16, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x63, 0x68, - 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x71, - 0x75, 0x6f, 0x74, 0x65, 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, 0x63, - 0x74, 0x65, 0x72, 0x1a, 0x37, 0x0a, 0x0a, 0x4a, 0x53, 0x4f, 0x4e, 0x4f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x22, 0x29, 0x0a, 0x0d, - 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x64, 0x53, 0x74, 0x72, 0x69, 0x70, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x55, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x22, 0xae, - 0x01, 0x0a, 0x1a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, - 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, - 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, - 0x69, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, - 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6c, - 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, - 0x72, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x63, 0x72, 0x63, 0x12, 0x10, 0x0a, - 0x03, 0x74, 0x74, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x32, - 0x9f, 0x22, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x6e, 0x52, 0x12, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x65, 0x0a, 0x14, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, + 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, + 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x4e, 0x0a, 0x06, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x18, 0x0a, 0x07, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x6e, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0xd5, 0x05, 0x0a, + 0x12, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, + 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x57, + 0x0a, 0x09, 0x63, 0x73, 0x76, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x3a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x53, 0x56, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, + 0x73, 0x76, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x5a, 0x0a, 0x0a, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, + 0x53, 0x4f, 0x4e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x12, 0x63, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x5f, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x61, + 0x72, 0x71, 0x75, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x71, + 0x75, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x1a, 0xc8, 0x02, 0x0a, 0x08, 0x43, 0x53, 0x56, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x68, 0x61, + 0x72, 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x71, + 0x75, 0x6f, 0x74, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x12, 0x34, + 0x0a, 0x16, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x63, + 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, + 0x71, 0x75, 0x6f, 0x74, 0x65, 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, + 0x63, 0x74, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x12, 0x41, 0x0a, 0x1d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x64, + 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, + 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x51, 0x75, + 0x6f, 0x74, 0x65, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x65, 0x72, 0x1a, 0x1f, 0x0a, 0x09, 0x4a, 0x53, 0x4f, 0x4e, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x1a, 0x0e, 0x0a, 0x0c, 0x50, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x49, + 0x6e, 0x70, 0x75, 0x74, 0x1a, 0xf1, 0x03, 0x0a, 0x13, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, + 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0a, + 0x63, 0x73, 0x76, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x3c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x53, 0x56, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x09, + 0x63, 0x73, 0x76, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x5e, 0x0a, 0x0b, 0x6a, 0x73, 0x6f, + 0x6e, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, + 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x6a, + 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x1a, 0xe3, 0x01, 0x0a, 0x09, 0x43, 0x53, + 0x56, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x71, 0x75, 0x6f, 0x74, 0x65, + 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x71, + 0x75, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, 0x65, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x64, + 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x29, + 0x0a, 0x10, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x43, + 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x6f, 0x65, 0x72, 0x12, 0x34, 0x0a, 0x16, 0x71, 0x75, 0x6f, + 0x74, 0x65, 0x5f, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x61, 0x63, + 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x71, 0x75, 0x6f, 0x74, 0x65, + 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x43, 0x68, 0x61, 0x72, 0x61, 0x63, 0x74, 0x65, 0x72, 0x1a, + 0x37, 0x0a, 0x0a, 0x4a, 0x53, 0x4f, 0x4e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x29, 0x0a, + 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, + 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x44, + 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x22, 0x29, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, + 0x69, 0x65, 0x64, 0x53, 0x74, 0x72, 0x69, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x73, 0x22, 0x55, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, + 0x64, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, + 0x09, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x08, 0x6e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x22, 0xae, 0x01, 0x0a, 0x1a, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x65, + 0x64, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x65, + 0x65, 0x64, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, + 0x69, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6c, 0x61, 0x73, 0x74, 0x4d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x72, 0x63, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x63, 0x72, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x22, 0x46, 0x0a, 0x0b, 0x50, + 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x22, 0x7a, 0x0a, 0x0c, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x20, 0x0a, + 0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x32, + 0xd5, 0x23, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x5c, 0x0a, 0x0b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x24, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, @@ -5833,7 +6237,7 @@ var file_volume_server_proto_rawDesc = []byte{ 0x75, 0x6d, 0x65, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x76, 0x0a, 0x13, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x12, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, @@ -5841,264 +6245,275 @@ var file_volume_server_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, - 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, - 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x56, 0x61, 0x63, 0x75, 0x75, - 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x2c, + 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x2b, 0x2e, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, + 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x56, 0x61, 0x63, + 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, + 0x12, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6c, 0x65, 0x61, - 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b, 0x0a, - 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x0e, 0x41, 0x6c, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x27, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, - 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x6b, 0x0a, 0x10, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, - 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, - 0x0a, 0x15, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x6b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, + 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x0e, + 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x27, + 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, + 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x6b, 0x0a, 0x10, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, + 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x53, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x79, 0x6e, 0x63, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x7c, 0x0a, 0x15, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, + 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, + 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5c, + 0x0a, 0x0b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x24, 0x2e, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x0d, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x2e, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, + 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x5f, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x12, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, + 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x70, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x0b, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x24, 0x2e, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x0d, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x2e, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x55, 0x6e, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5f, - 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x25, - 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, - 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, - 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, - 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, - 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, + 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, + 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x28, 0x2e, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x5f, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, - 0x61, 0x72, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, + 0x12, 0x23, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, + 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, + 0x77, 0x0a, 0x14, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x69, 0x6c, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x08, 0x43, 0x6f, 0x70, 0x79, + 0x46, 0x69, 0x6c, 0x65, 0x12, 0x21, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x46, + 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, + 0x65, 0x0a, 0x0e, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, + 0x62, 0x12, 0x27, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, + 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0f, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, + 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x67, 0x0a, 0x0e, 0x52, 0x65, 0x61, 0x64, 0x41, 0x6c, 0x6c, 0x4e, 0x65, 0x65, 0x64, 0x6c, + 0x65, 0x73, 0x12, 0x27, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x41, 0x6c, 0x6c, 0x4e, 0x65, 0x65, + 0x64, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, + 0x65, 0x61, 0x64, 0x41, 0x6c, 0x6c, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6d, 0x0a, 0x10, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x29, 0x2e, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x5f, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x25, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x59, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x23, + 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x70, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x77, 0x0a, 0x14, 0x52, - 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x46, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x08, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, - 0x12, 0x21, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x0e, 0x52, - 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x27, 0x2e, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x4e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0f, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, - 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x28, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x42, 0x6c, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x10, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, - 0x12, 0x29, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, 0x6f, + 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, + 0x69, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, - 0x72, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, - 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, - 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x61, 0x69, 0x6c, 0x52, 0x65, 0x63, 0x65, - 0x69, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7d, - 0x0a, 0x16, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, - 0x15, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, - 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7d, 0x0a, 0x16, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, + 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x12, - 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, - 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x77, 0x0a, 0x14, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x15, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, + 0x69, 0x6c, 0x64, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, - 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, - 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x15, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x12, 0x2a, 0x2e, 0x76, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, - 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, + 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x43, 0x6f, 0x70, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x77, 0x0a, 0x14, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x15, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x2e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x55, 0x6e, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x12, 0x2a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, + 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7d, 0x0a, 0x16, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a, 0x19, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, - 0x74, 0x54, 0x6f, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x42, 0x6c, 0x6f, 0x62, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7d, 0x0a, 0x16, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x12, 0x2f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x63, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x54, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a, 0x19, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x54, 0x6f, - 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, - 0x61, 0x74, 0x54, 0x6f, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x8e, 0x01, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, - 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x12, 0x34, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x35, 0x2e, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, - 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, + 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x54, 0x6f, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x76, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x54, + 0x6f, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x30, 0x01, 0x12, 0x8e, 0x01, 0x0a, 0x1b, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, + 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x12, 0x34, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, + 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x35, 0x2e, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x54, 0x69, 0x65, 0x72, 0x4d, 0x6f, 0x76, 0x65, 0x44, 0x61, 0x74, 0x46, 0x72, + 0x6f, 0x6d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6e, 0x0a, 0x11, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x12, - 0x2a, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, - 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x46, 0x65, - 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, - 0x65, 0x12, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x4c, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, - 0x72, 0x69, 0x65, 0x64, 0x53, 0x74, 0x72, 0x69, 0x70, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, - 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, - 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6e, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x12, 0x2a, 0x2e, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, 0x13, 0x46, 0x65, 0x74, 0x63, 0x68, + 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x12, 0x2c, + 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, + 0x65, 0x65, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x46, 0x65, 0x74, 0x63, 0x68, 0x41, 0x6e, 0x64, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4e, 0x65, 0x65, + 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, + 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, + 0x64, 0x53, 0x74, 0x72, 0x69, 0x70, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x12, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x2b, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, - 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x76, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4e, 0x65, 0x65, 0x64, 0x6c, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x47, + 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, + 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, + 0x62, 0x2f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -6113,7 +6528,7 @@ func file_volume_server_proto_rawDescGZIP() []byte { return file_volume_server_proto_rawDescData } -var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 92) +var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 97) var file_volume_server_proto_goTypes = []interface{}{ (*BatchDeleteRequest)(nil), // 0: volume_server_pb.BatchDeleteRequest (*BatchDeleteResponse)(nil), // 1: volume_server_pb.BatchDeleteResponse @@ -6157,157 +6572,167 @@ var file_volume_server_proto_goTypes = []interface{}{ (*ReadNeedleBlobResponse)(nil), // 39: volume_server_pb.ReadNeedleBlobResponse (*WriteNeedleBlobRequest)(nil), // 40: volume_server_pb.WriteNeedleBlobRequest (*WriteNeedleBlobResponse)(nil), // 41: volume_server_pb.WriteNeedleBlobResponse - (*VolumeTailSenderRequest)(nil), // 42: volume_server_pb.VolumeTailSenderRequest - (*VolumeTailSenderResponse)(nil), // 43: volume_server_pb.VolumeTailSenderResponse - (*VolumeTailReceiverRequest)(nil), // 44: volume_server_pb.VolumeTailReceiverRequest - (*VolumeTailReceiverResponse)(nil), // 45: volume_server_pb.VolumeTailReceiverResponse - (*VolumeEcShardsGenerateRequest)(nil), // 46: volume_server_pb.VolumeEcShardsGenerateRequest - (*VolumeEcShardsGenerateResponse)(nil), // 47: volume_server_pb.VolumeEcShardsGenerateResponse - (*VolumeEcShardsRebuildRequest)(nil), // 48: volume_server_pb.VolumeEcShardsRebuildRequest - (*VolumeEcShardsRebuildResponse)(nil), // 49: volume_server_pb.VolumeEcShardsRebuildResponse - (*VolumeEcShardsCopyRequest)(nil), // 50: volume_server_pb.VolumeEcShardsCopyRequest - (*VolumeEcShardsCopyResponse)(nil), // 51: volume_server_pb.VolumeEcShardsCopyResponse - (*VolumeEcShardsDeleteRequest)(nil), // 52: volume_server_pb.VolumeEcShardsDeleteRequest - (*VolumeEcShardsDeleteResponse)(nil), // 53: volume_server_pb.VolumeEcShardsDeleteResponse - (*VolumeEcShardsMountRequest)(nil), // 54: volume_server_pb.VolumeEcShardsMountRequest - (*VolumeEcShardsMountResponse)(nil), // 55: volume_server_pb.VolumeEcShardsMountResponse - (*VolumeEcShardsUnmountRequest)(nil), // 56: volume_server_pb.VolumeEcShardsUnmountRequest - (*VolumeEcShardsUnmountResponse)(nil), // 57: volume_server_pb.VolumeEcShardsUnmountResponse - (*VolumeEcShardReadRequest)(nil), // 58: volume_server_pb.VolumeEcShardReadRequest - (*VolumeEcShardReadResponse)(nil), // 59: volume_server_pb.VolumeEcShardReadResponse - (*VolumeEcBlobDeleteRequest)(nil), // 60: volume_server_pb.VolumeEcBlobDeleteRequest - (*VolumeEcBlobDeleteResponse)(nil), // 61: volume_server_pb.VolumeEcBlobDeleteResponse - (*VolumeEcShardsToVolumeRequest)(nil), // 62: volume_server_pb.VolumeEcShardsToVolumeRequest - (*VolumeEcShardsToVolumeResponse)(nil), // 63: volume_server_pb.VolumeEcShardsToVolumeResponse - (*ReadVolumeFileStatusRequest)(nil), // 64: volume_server_pb.ReadVolumeFileStatusRequest - (*ReadVolumeFileStatusResponse)(nil), // 65: volume_server_pb.ReadVolumeFileStatusResponse - (*DiskStatus)(nil), // 66: volume_server_pb.DiskStatus - (*MemStatus)(nil), // 67: volume_server_pb.MemStatus - (*RemoteFile)(nil), // 68: volume_server_pb.RemoteFile - (*VolumeInfo)(nil), // 69: volume_server_pb.VolumeInfo - (*VolumeTierMoveDatToRemoteRequest)(nil), // 70: volume_server_pb.VolumeTierMoveDatToRemoteRequest - (*VolumeTierMoveDatToRemoteResponse)(nil), // 71: volume_server_pb.VolumeTierMoveDatToRemoteResponse - (*VolumeTierMoveDatFromRemoteRequest)(nil), // 72: volume_server_pb.VolumeTierMoveDatFromRemoteRequest - (*VolumeTierMoveDatFromRemoteResponse)(nil), // 73: volume_server_pb.VolumeTierMoveDatFromRemoteResponse - (*VolumeServerStatusRequest)(nil), // 74: volume_server_pb.VolumeServerStatusRequest - (*VolumeServerStatusResponse)(nil), // 75: volume_server_pb.VolumeServerStatusResponse - (*VolumeServerLeaveRequest)(nil), // 76: volume_server_pb.VolumeServerLeaveRequest - (*VolumeServerLeaveResponse)(nil), // 77: volume_server_pb.VolumeServerLeaveResponse - (*FetchAndWriteNeedleRequest)(nil), // 78: volume_server_pb.FetchAndWriteNeedleRequest - (*FetchAndWriteNeedleResponse)(nil), // 79: volume_server_pb.FetchAndWriteNeedleResponse - (*QueryRequest)(nil), // 80: volume_server_pb.QueryRequest - (*QueriedStripe)(nil), // 81: volume_server_pb.QueriedStripe - (*VolumeNeedleStatusRequest)(nil), // 82: volume_server_pb.VolumeNeedleStatusRequest - (*VolumeNeedleStatusResponse)(nil), // 83: volume_server_pb.VolumeNeedleStatusResponse - (*QueryRequest_Filter)(nil), // 84: volume_server_pb.QueryRequest.Filter - (*QueryRequest_InputSerialization)(nil), // 85: volume_server_pb.QueryRequest.InputSerialization - (*QueryRequest_OutputSerialization)(nil), // 86: volume_server_pb.QueryRequest.OutputSerialization - (*QueryRequest_InputSerialization_CSVInput)(nil), // 87: volume_server_pb.QueryRequest.InputSerialization.CSVInput - (*QueryRequest_InputSerialization_JSONInput)(nil), // 88: volume_server_pb.QueryRequest.InputSerialization.JSONInput - (*QueryRequest_InputSerialization_ParquetInput)(nil), // 89: volume_server_pb.QueryRequest.InputSerialization.ParquetInput - (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 90: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 91: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput - (*remote_pb.RemoteConf)(nil), // 92: remote_pb.RemoteConf - (*remote_pb.RemoteStorageLocation)(nil), // 93: remote_pb.RemoteStorageLocation + (*ReadAllNeedlesRequest)(nil), // 42: volume_server_pb.ReadAllNeedlesRequest + (*ReadAllNeedlesResponse)(nil), // 43: volume_server_pb.ReadAllNeedlesResponse + (*VolumeTailSenderRequest)(nil), // 44: volume_server_pb.VolumeTailSenderRequest + (*VolumeTailSenderResponse)(nil), // 45: volume_server_pb.VolumeTailSenderResponse + (*VolumeTailReceiverRequest)(nil), // 46: volume_server_pb.VolumeTailReceiverRequest + (*VolumeTailReceiverResponse)(nil), // 47: volume_server_pb.VolumeTailReceiverResponse + (*VolumeEcShardsGenerateRequest)(nil), // 48: volume_server_pb.VolumeEcShardsGenerateRequest + (*VolumeEcShardsGenerateResponse)(nil), // 49: volume_server_pb.VolumeEcShardsGenerateResponse + (*VolumeEcShardsRebuildRequest)(nil), // 50: volume_server_pb.VolumeEcShardsRebuildRequest + (*VolumeEcShardsRebuildResponse)(nil), // 51: volume_server_pb.VolumeEcShardsRebuildResponse + (*VolumeEcShardsCopyRequest)(nil), // 52: volume_server_pb.VolumeEcShardsCopyRequest + (*VolumeEcShardsCopyResponse)(nil), // 53: volume_server_pb.VolumeEcShardsCopyResponse + (*VolumeEcShardsDeleteRequest)(nil), // 54: volume_server_pb.VolumeEcShardsDeleteRequest + (*VolumeEcShardsDeleteResponse)(nil), // 55: volume_server_pb.VolumeEcShardsDeleteResponse + (*VolumeEcShardsMountRequest)(nil), // 56: volume_server_pb.VolumeEcShardsMountRequest + (*VolumeEcShardsMountResponse)(nil), // 57: volume_server_pb.VolumeEcShardsMountResponse + (*VolumeEcShardsUnmountRequest)(nil), // 58: volume_server_pb.VolumeEcShardsUnmountRequest + (*VolumeEcShardsUnmountResponse)(nil), // 59: volume_server_pb.VolumeEcShardsUnmountResponse + (*VolumeEcShardReadRequest)(nil), // 60: volume_server_pb.VolumeEcShardReadRequest + (*VolumeEcShardReadResponse)(nil), // 61: volume_server_pb.VolumeEcShardReadResponse + (*VolumeEcBlobDeleteRequest)(nil), // 62: volume_server_pb.VolumeEcBlobDeleteRequest + (*VolumeEcBlobDeleteResponse)(nil), // 63: volume_server_pb.VolumeEcBlobDeleteResponse + (*VolumeEcShardsToVolumeRequest)(nil), // 64: volume_server_pb.VolumeEcShardsToVolumeRequest + (*VolumeEcShardsToVolumeResponse)(nil), // 65: volume_server_pb.VolumeEcShardsToVolumeResponse + (*ReadVolumeFileStatusRequest)(nil), // 66: volume_server_pb.ReadVolumeFileStatusRequest + (*ReadVolumeFileStatusResponse)(nil), // 67: volume_server_pb.ReadVolumeFileStatusResponse + (*DiskStatus)(nil), // 68: volume_server_pb.DiskStatus + (*MemStatus)(nil), // 69: volume_server_pb.MemStatus + (*RemoteFile)(nil), // 70: volume_server_pb.RemoteFile + (*VolumeInfo)(nil), // 71: volume_server_pb.VolumeInfo + (*VolumeTierMoveDatToRemoteRequest)(nil), // 72: volume_server_pb.VolumeTierMoveDatToRemoteRequest + (*VolumeTierMoveDatToRemoteResponse)(nil), // 73: volume_server_pb.VolumeTierMoveDatToRemoteResponse + (*VolumeTierMoveDatFromRemoteRequest)(nil), // 74: volume_server_pb.VolumeTierMoveDatFromRemoteRequest + (*VolumeTierMoveDatFromRemoteResponse)(nil), // 75: volume_server_pb.VolumeTierMoveDatFromRemoteResponse + (*VolumeServerStatusRequest)(nil), // 76: volume_server_pb.VolumeServerStatusRequest + (*VolumeServerStatusResponse)(nil), // 77: volume_server_pb.VolumeServerStatusResponse + (*VolumeServerLeaveRequest)(nil), // 78: volume_server_pb.VolumeServerLeaveRequest + (*VolumeServerLeaveResponse)(nil), // 79: volume_server_pb.VolumeServerLeaveResponse + (*FetchAndWriteNeedleRequest)(nil), // 80: volume_server_pb.FetchAndWriteNeedleRequest + (*FetchAndWriteNeedleResponse)(nil), // 81: volume_server_pb.FetchAndWriteNeedleResponse + (*QueryRequest)(nil), // 82: volume_server_pb.QueryRequest + (*QueriedStripe)(nil), // 83: volume_server_pb.QueriedStripe + (*VolumeNeedleStatusRequest)(nil), // 84: volume_server_pb.VolumeNeedleStatusRequest + (*VolumeNeedleStatusResponse)(nil), // 85: volume_server_pb.VolumeNeedleStatusResponse + (*PingRequest)(nil), // 86: volume_server_pb.PingRequest + (*PingResponse)(nil), // 87: volume_server_pb.PingResponse + (*FetchAndWriteNeedleRequest_Replica)(nil), // 88: volume_server_pb.FetchAndWriteNeedleRequest.Replica + (*QueryRequest_Filter)(nil), // 89: volume_server_pb.QueryRequest.Filter + (*QueryRequest_InputSerialization)(nil), // 90: volume_server_pb.QueryRequest.InputSerialization + (*QueryRequest_OutputSerialization)(nil), // 91: volume_server_pb.QueryRequest.OutputSerialization + (*QueryRequest_InputSerialization_CSVInput)(nil), // 92: volume_server_pb.QueryRequest.InputSerialization.CSVInput + (*QueryRequest_InputSerialization_JSONInput)(nil), // 93: volume_server_pb.QueryRequest.InputSerialization.JSONInput + (*QueryRequest_InputSerialization_ParquetInput)(nil), // 94: volume_server_pb.QueryRequest.InputSerialization.ParquetInput + (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 95: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 96: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + (*remote_pb.RemoteConf)(nil), // 97: remote_pb.RemoteConf + (*remote_pb.RemoteStorageLocation)(nil), // 98: remote_pb.RemoteStorageLocation } var file_volume_server_proto_depIdxs = []int32{ 2, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult - 68, // 1: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile - 66, // 2: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus - 67, // 3: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus - 92, // 4: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf - 93, // 5: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation - 84, // 6: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter - 85, // 7: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization - 86, // 8: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization - 87, // 9: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput - 88, // 10: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput - 89, // 11: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput - 90, // 12: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - 91, // 13: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput - 0, // 14: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest - 4, // 15: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest - 6, // 16: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest - 8, // 17: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest - 10, // 18: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest - 12, // 19: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest - 14, // 20: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest - 16, // 21: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest - 18, // 22: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest - 20, // 23: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest - 22, // 24: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest - 24, // 25: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest - 26, // 26: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest - 28, // 27: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest - 30, // 28: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest - 32, // 29: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest - 34, // 30: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest - 64, // 31: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest - 36, // 32: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest - 38, // 33: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest - 40, // 34: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest - 42, // 35: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest - 44, // 36: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest - 46, // 37: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest - 48, // 38: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest - 50, // 39: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest - 52, // 40: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest - 54, // 41: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest - 56, // 42: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest - 58, // 43: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest - 60, // 44: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest - 62, // 45: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest - 70, // 46: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest - 72, // 47: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest - 74, // 48: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest - 76, // 49: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest - 78, // 50: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest - 80, // 51: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest - 82, // 52: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest - 1, // 53: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse - 5, // 54: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse - 7, // 55: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse - 9, // 56: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse - 11, // 57: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse - 13, // 58: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse - 15, // 59: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse - 17, // 60: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse - 19, // 61: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse - 21, // 62: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse - 23, // 63: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse - 25, // 64: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse - 27, // 65: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse - 29, // 66: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse - 31, // 67: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse - 33, // 68: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse - 35, // 69: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse - 65, // 70: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse - 37, // 71: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse - 39, // 72: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse - 41, // 73: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse - 43, // 74: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse - 45, // 75: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse - 47, // 76: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse - 49, // 77: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse - 51, // 78: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse - 53, // 79: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse - 55, // 80: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse - 57, // 81: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse - 59, // 82: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse - 61, // 83: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse - 63, // 84: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse - 71, // 85: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse - 73, // 86: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse - 75, // 87: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse - 77, // 88: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse - 79, // 89: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse - 81, // 90: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe - 83, // 91: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse - 53, // [53:92] is the sub-list for method output_type - 14, // [14:53] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 70, // 1: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile + 68, // 2: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus + 69, // 3: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus + 88, // 4: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica + 97, // 5: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf + 98, // 6: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation + 89, // 7: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter + 90, // 8: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization + 91, // 9: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization + 92, // 10: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput + 93, // 11: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput + 94, // 12: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput + 95, // 13: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + 96, // 14: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + 0, // 15: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest + 4, // 16: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest + 6, // 17: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest + 8, // 18: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest + 10, // 19: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest + 12, // 20: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest + 14, // 21: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest + 16, // 22: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest + 18, // 23: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest + 20, // 24: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest + 22, // 25: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest + 24, // 26: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest + 26, // 27: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest + 28, // 28: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest + 30, // 29: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest + 32, // 30: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest + 34, // 31: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest + 66, // 32: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest + 36, // 33: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest + 38, // 34: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest + 40, // 35: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest + 42, // 36: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest + 44, // 37: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest + 46, // 38: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest + 48, // 39: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest + 50, // 40: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest + 52, // 41: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest + 54, // 42: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest + 56, // 43: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest + 58, // 44: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest + 60, // 45: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest + 62, // 46: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest + 64, // 47: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest + 72, // 48: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest + 74, // 49: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest + 76, // 50: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest + 78, // 51: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest + 80, // 52: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest + 82, // 53: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest + 84, // 54: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest + 86, // 55: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest + 1, // 56: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse + 5, // 57: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse + 7, // 58: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse + 9, // 59: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse + 11, // 60: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse + 13, // 61: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse + 15, // 62: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse + 17, // 63: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse + 19, // 64: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse + 21, // 65: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse + 23, // 66: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse + 25, // 67: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse + 27, // 68: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse + 29, // 69: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse + 31, // 70: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse + 33, // 71: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse + 35, // 72: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse + 67, // 73: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse + 37, // 74: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse + 39, // 75: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse + 41, // 76: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse + 43, // 77: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse + 45, // 78: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse + 47, // 79: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse + 49, // 80: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse + 51, // 81: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse + 53, // 82: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse + 55, // 83: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse + 57, // 84: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse + 59, // 85: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse + 61, // 86: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse + 63, // 87: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse + 65, // 88: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse + 73, // 89: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse + 75, // 90: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse + 77, // 91: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse + 79, // 92: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse + 81, // 93: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse + 83, // 94: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe + 85, // 95: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse + 87, // 96: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse + 56, // [56:97] is the sub-list for method output_type + 15, // [15:56] is the sub-list for method input_type + 15, // [15:15] is the sub-list for extension type_name + 15, // [15:15] is the sub-list for extension extendee + 0, // [0:15] is the sub-list for field type_name } func init() { file_volume_server_proto_init() } @@ -6821,7 +7246,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTailSenderRequest); i { + switch v := v.(*ReadAllNeedlesRequest); i { case 0: return &v.state case 1: @@ -6833,7 +7258,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTailSenderResponse); i { + switch v := v.(*ReadAllNeedlesResponse); i { case 0: return &v.state case 1: @@ -6845,7 +7270,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTailReceiverRequest); i { + switch v := v.(*VolumeTailSenderRequest); i { case 0: return &v.state case 1: @@ -6857,7 +7282,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTailReceiverResponse); i { + switch v := v.(*VolumeTailSenderResponse); i { case 0: return &v.state case 1: @@ -6869,7 +7294,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsGenerateRequest); i { + switch v := v.(*VolumeTailReceiverRequest); i { case 0: return &v.state case 1: @@ -6881,7 +7306,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsGenerateResponse); i { + switch v := v.(*VolumeTailReceiverResponse); i { case 0: return &v.state case 1: @@ -6893,7 +7318,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsRebuildRequest); i { + switch v := v.(*VolumeEcShardsGenerateRequest); i { case 0: return &v.state case 1: @@ -6905,7 +7330,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsRebuildResponse); i { + switch v := v.(*VolumeEcShardsGenerateResponse); i { case 0: return &v.state case 1: @@ -6917,7 +7342,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsCopyRequest); i { + switch v := v.(*VolumeEcShardsRebuildRequest); i { case 0: return &v.state case 1: @@ -6929,7 +7354,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsCopyResponse); i { + switch v := v.(*VolumeEcShardsRebuildResponse); i { case 0: return &v.state case 1: @@ -6941,7 +7366,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsDeleteRequest); i { + switch v := v.(*VolumeEcShardsCopyRequest); i { case 0: return &v.state case 1: @@ -6953,7 +7378,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsDeleteResponse); i { + switch v := v.(*VolumeEcShardsCopyResponse); i { case 0: return &v.state case 1: @@ -6965,7 +7390,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsMountRequest); i { + switch v := v.(*VolumeEcShardsDeleteRequest); i { case 0: return &v.state case 1: @@ -6977,7 +7402,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsMountResponse); i { + switch v := v.(*VolumeEcShardsDeleteResponse); i { case 0: return &v.state case 1: @@ -6989,7 +7414,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsUnmountRequest); i { + switch v := v.(*VolumeEcShardsMountRequest); i { case 0: return &v.state case 1: @@ -7001,7 +7426,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsUnmountResponse); i { + switch v := v.(*VolumeEcShardsMountResponse); i { case 0: return &v.state case 1: @@ -7013,7 +7438,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardReadRequest); i { + switch v := v.(*VolumeEcShardsUnmountRequest); i { case 0: return &v.state case 1: @@ -7025,7 +7450,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardReadResponse); i { + switch v := v.(*VolumeEcShardsUnmountResponse); i { case 0: return &v.state case 1: @@ -7037,7 +7462,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcBlobDeleteRequest); i { + switch v := v.(*VolumeEcShardReadRequest); i { case 0: return &v.state case 1: @@ -7049,7 +7474,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcBlobDeleteResponse); i { + switch v := v.(*VolumeEcShardReadResponse); i { case 0: return &v.state case 1: @@ -7061,7 +7486,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsToVolumeRequest); i { + switch v := v.(*VolumeEcBlobDeleteRequest); i { case 0: return &v.state case 1: @@ -7073,7 +7498,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeEcShardsToVolumeResponse); i { + switch v := v.(*VolumeEcBlobDeleteResponse); i { case 0: return &v.state case 1: @@ -7085,7 +7510,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadVolumeFileStatusRequest); i { + switch v := v.(*VolumeEcShardsToVolumeRequest); i { case 0: return &v.state case 1: @@ -7097,7 +7522,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadVolumeFileStatusResponse); i { + switch v := v.(*VolumeEcShardsToVolumeResponse); i { case 0: return &v.state case 1: @@ -7109,7 +7534,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DiskStatus); i { + switch v := v.(*ReadVolumeFileStatusRequest); i { case 0: return &v.state case 1: @@ -7121,7 +7546,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MemStatus); i { + switch v := v.(*ReadVolumeFileStatusResponse); i { case 0: return &v.state case 1: @@ -7133,7 +7558,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoteFile); i { + switch v := v.(*DiskStatus); i { case 0: return &v.state case 1: @@ -7145,7 +7570,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeInfo); i { + switch v := v.(*MemStatus); i { case 0: return &v.state case 1: @@ -7157,7 +7582,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTierMoveDatToRemoteRequest); i { + switch v := v.(*RemoteFile); i { case 0: return &v.state case 1: @@ -7169,7 +7594,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTierMoveDatToRemoteResponse); i { + switch v := v.(*VolumeInfo); i { case 0: return &v.state case 1: @@ -7181,7 +7606,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTierMoveDatFromRemoteRequest); i { + switch v := v.(*VolumeTierMoveDatToRemoteRequest); i { case 0: return &v.state case 1: @@ -7193,7 +7618,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeTierMoveDatFromRemoteResponse); i { + switch v := v.(*VolumeTierMoveDatToRemoteResponse); i { case 0: return &v.state case 1: @@ -7205,7 +7630,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeServerStatusRequest); i { + switch v := v.(*VolumeTierMoveDatFromRemoteRequest); i { case 0: return &v.state case 1: @@ -7217,7 +7642,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeServerStatusResponse); i { + switch v := v.(*VolumeTierMoveDatFromRemoteResponse); i { case 0: return &v.state case 1: @@ -7229,7 +7654,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeServerLeaveRequest); i { + switch v := v.(*VolumeServerStatusRequest); i { case 0: return &v.state case 1: @@ -7241,7 +7666,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeServerLeaveResponse); i { + switch v := v.(*VolumeServerStatusResponse); i { case 0: return &v.state case 1: @@ -7253,7 +7678,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FetchAndWriteNeedleRequest); i { + switch v := v.(*VolumeServerLeaveRequest); i { case 0: return &v.state case 1: @@ -7265,7 +7690,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FetchAndWriteNeedleResponse); i { + switch v := v.(*VolumeServerLeaveResponse); i { case 0: return &v.state case 1: @@ -7277,7 +7702,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest); i { + switch v := v.(*FetchAndWriteNeedleRequest); i { case 0: return &v.state case 1: @@ -7289,7 +7714,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueriedStripe); i { + switch v := v.(*FetchAndWriteNeedleResponse); i { case 0: return &v.state case 1: @@ -7301,7 +7726,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeNeedleStatusRequest); i { + switch v := v.(*QueryRequest); i { case 0: return &v.state case 1: @@ -7313,7 +7738,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeNeedleStatusResponse); i { + switch v := v.(*QueriedStripe); i { case 0: return &v.state case 1: @@ -7325,7 +7750,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_Filter); i { + switch v := v.(*VolumeNeedleStatusRequest); i { case 0: return &v.state case 1: @@ -7337,7 +7762,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_InputSerialization); i { + switch v := v.(*VolumeNeedleStatusResponse); i { case 0: return &v.state case 1: @@ -7349,7 +7774,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_OutputSerialization); i { + switch v := v.(*PingRequest); i { case 0: return &v.state case 1: @@ -7361,7 +7786,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_InputSerialization_CSVInput); i { + switch v := v.(*PingResponse); i { case 0: return &v.state case 1: @@ -7373,7 +7798,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_InputSerialization_JSONInput); i { + switch v := v.(*FetchAndWriteNeedleRequest_Replica); i { case 0: return &v.state case 1: @@ -7385,7 +7810,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_InputSerialization_ParquetInput); i { + switch v := v.(*QueryRequest_Filter); i { case 0: return &v.state case 1: @@ -7397,7 +7822,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_OutputSerialization_CSVOutput); i { + switch v := v.(*QueryRequest_InputSerialization); i { case 0: return &v.state case 1: @@ -7409,7 +7834,7 @@ func file_volume_server_proto_init() { } } file_volume_server_proto_msgTypes[91].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryRequest_OutputSerialization_JSONOutput); i { + switch v := v.(*QueryRequest_OutputSerialization); i { case 0: return &v.state case 1: @@ -7420,16 +7845,76 @@ func file_volume_server_proto_init() { return nil } } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_volume_server_proto_rawDesc, - NumEnums: 0, - NumMessages: 92, - NumExtensions: 0, - NumServices: 1, + file_volume_server_proto_msgTypes[92].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QueryRequest_InputSerialization_CSVInput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_volume_server_proto_msgTypes[93].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QueryRequest_InputSerialization_JSONInput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_volume_server_proto_msgTypes[94].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QueryRequest_InputSerialization_ParquetInput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_volume_server_proto_msgTypes[95].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QueryRequest_OutputSerialization_CSVOutput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_volume_server_proto_msgTypes[96].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QueryRequest_OutputSerialization_JSONOutput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_volume_server_proto_rawDesc, + NumEnums: 0, + NumMessages: 97, + NumExtensions: 0, + NumServices: 1, }, GoTypes: file_volume_server_proto_goTypes, DependencyIndexes: file_volume_server_proto_depIdxs, @@ -7440,1653 +7925,3 @@ func file_volume_server_proto_init() { file_volume_server_proto_goTypes = nil file_volume_server_proto_depIdxs = nil } - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// VolumeServerClient is the client API for VolumeServer service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type VolumeServerClient interface { - //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. - BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteResponse, error) - VacuumVolumeCheck(ctx context.Context, in *VacuumVolumeCheckRequest, opts ...grpc.CallOption) (*VacuumVolumeCheckResponse, error) - VacuumVolumeCompact(ctx context.Context, in *VacuumVolumeCompactRequest, opts ...grpc.CallOption) (*VacuumVolumeCompactResponse, error) - VacuumVolumeCommit(ctx context.Context, in *VacuumVolumeCommitRequest, opts ...grpc.CallOption) (*VacuumVolumeCommitResponse, error) - VacuumVolumeCleanup(ctx context.Context, in *VacuumVolumeCleanupRequest, opts ...grpc.CallOption) (*VacuumVolumeCleanupResponse, error) - DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) - AllocateVolume(ctx context.Context, in *AllocateVolumeRequest, opts ...grpc.CallOption) (*AllocateVolumeResponse, error) - VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error) - VolumeIncrementalCopy(ctx context.Context, in *VolumeIncrementalCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeIncrementalCopyClient, error) - VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) - VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error) - VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) - VolumeMarkReadonly(ctx context.Context, in *VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*VolumeMarkReadonlyResponse, error) - VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error) - VolumeConfigure(ctx context.Context, in *VolumeConfigureRequest, opts ...grpc.CallOption) (*VolumeConfigureResponse, error) - VolumeStatus(ctx context.Context, in *VolumeStatusRequest, opts ...grpc.CallOption) (*VolumeStatusResponse, error) - // copy the .idx .dat files, and mount this volume - VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (*VolumeCopyResponse, error) - ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error) - CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) - ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error) - WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) - VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error) - VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error) - // erasure coding - VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) - VolumeEcShardsRebuild(ctx context.Context, in *VolumeEcShardsRebuildRequest, opts ...grpc.CallOption) (*VolumeEcShardsRebuildResponse, error) - VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) - VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) - VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) - VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) - VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) - VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) - VolumeEcShardsToVolume(ctx context.Context, in *VolumeEcShardsToVolumeRequest, opts ...grpc.CallOption) (*VolumeEcShardsToVolumeResponse, error) - // tiered storage - VolumeTierMoveDatToRemote(ctx context.Context, in *VolumeTierMoveDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatToRemoteClient, error) - VolumeTierMoveDatFromRemote(ctx context.Context, in *VolumeTierMoveDatFromRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatFromRemoteClient, error) - VolumeServerStatus(ctx context.Context, in *VolumeServerStatusRequest, opts ...grpc.CallOption) (*VolumeServerStatusResponse, error) - VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error) - // remote storage - FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error) - // query - Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) - VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error) -} - -type volumeServerClient struct { - cc grpc.ClientConnInterface -} - -func NewVolumeServerClient(cc grpc.ClientConnInterface) VolumeServerClient { - return &volumeServerClient{cc} -} - -func (c *volumeServerClient) BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteResponse, error) { - out := new(BatchDeleteResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/BatchDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VacuumVolumeCheck(ctx context.Context, in *VacuumVolumeCheckRequest, opts ...grpc.CallOption) (*VacuumVolumeCheckResponse, error) { - out := new(VacuumVolumeCheckResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCheck", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VacuumVolumeCompact(ctx context.Context, in *VacuumVolumeCompactRequest, opts ...grpc.CallOption) (*VacuumVolumeCompactResponse, error) { - out := new(VacuumVolumeCompactResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCompact", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VacuumVolumeCommit(ctx context.Context, in *VacuumVolumeCommitRequest, opts ...grpc.CallOption) (*VacuumVolumeCommitResponse, error) { - out := new(VacuumVolumeCommitResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCommit", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VacuumVolumeCleanup(ctx context.Context, in *VacuumVolumeCleanupRequest, opts ...grpc.CallOption) (*VacuumVolumeCleanupResponse, error) { - out := new(VacuumVolumeCleanupResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCleanup", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) { - out := new(DeleteCollectionResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/DeleteCollection", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) AllocateVolume(ctx context.Context, in *AllocateVolumeRequest, opts ...grpc.CallOption) (*AllocateVolumeResponse, error) { - out := new(AllocateVolumeResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/AllocateVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error) { - out := new(VolumeSyncStatusResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeIncrementalCopy(ctx context.Context, in *VolumeIncrementalCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeIncrementalCopyClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[0], "/volume_server_pb.VolumeServer/VolumeIncrementalCopy", opts...) - if err != nil { - return nil, err - } - x := &volumeServerVolumeIncrementalCopyClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_VolumeIncrementalCopyClient interface { - Recv() (*VolumeIncrementalCopyResponse, error) - grpc.ClientStream -} - -type volumeServerVolumeIncrementalCopyClient struct { - grpc.ClientStream -} - -func (x *volumeServerVolumeIncrementalCopyClient) Recv() (*VolumeIncrementalCopyResponse, error) { - m := new(VolumeIncrementalCopyResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) { - out := new(VolumeMountResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMount", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error) { - out := new(VolumeUnmountResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeUnmount", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) { - out := new(VolumeDeleteResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeMarkReadonly(ctx context.Context, in *VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*VolumeMarkReadonlyResponse, error) { - out := new(VolumeMarkReadonlyResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMarkReadonly", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error) { - out := new(VolumeMarkWritableResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMarkWritable", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeConfigure(ctx context.Context, in *VolumeConfigureRequest, opts ...grpc.CallOption) (*VolumeConfigureResponse, error) { - out := new(VolumeConfigureResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeConfigure", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeStatus(ctx context.Context, in *VolumeStatusRequest, opts ...grpc.CallOption) (*VolumeStatusResponse, error) { - out := new(VolumeStatusResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (*VolumeCopyResponse, error) { - out := new(VolumeCopyResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeCopy", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error) { - out := new(ReadVolumeFileStatusResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadVolumeFileStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[1], "/volume_server_pb.VolumeServer/CopyFile", opts...) - if err != nil { - return nil, err - } - x := &volumeServerCopyFileClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_CopyFileClient interface { - Recv() (*CopyFileResponse, error) - grpc.ClientStream -} - -type volumeServerCopyFileClient struct { - grpc.ClientStream -} - -func (x *volumeServerCopyFileClient) Recv() (*CopyFileResponse, error) { - m := new(CopyFileResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error) { - out := new(ReadNeedleBlobResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadNeedleBlob", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) { - out := new(WriteNeedleBlobResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/WriteNeedleBlob", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[2], "/volume_server_pb.VolumeServer/VolumeTailSender", opts...) - if err != nil { - return nil, err - } - x := &volumeServerVolumeTailSenderClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_VolumeTailSenderClient interface { - Recv() (*VolumeTailSenderResponse, error) - grpc.ClientStream -} - -type volumeServerVolumeTailSenderClient struct { - grpc.ClientStream -} - -func (x *volumeServerVolumeTailSenderClient) Recv() (*VolumeTailSenderResponse, error) { - m := new(VolumeTailSenderResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error) { - out := new(VolumeTailReceiverResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeTailReceiver", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) { - out := new(VolumeEcShardsGenerateResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsRebuild(ctx context.Context, in *VolumeEcShardsRebuildRequest, opts ...grpc.CallOption) (*VolumeEcShardsRebuildResponse, error) { - out := new(VolumeEcShardsRebuildResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) { - out := new(VolumeEcShardsCopyResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) { - out := new(VolumeEcShardsDeleteResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) { - out := new(VolumeEcShardsMountResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsMount", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) { - out := new(VolumeEcShardsUnmountResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[3], "/volume_server_pb.VolumeServer/VolumeEcShardRead", opts...) - if err != nil { - return nil, err - } - x := &volumeServerVolumeEcShardReadClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_VolumeEcShardReadClient interface { - Recv() (*VolumeEcShardReadResponse, error) - grpc.ClientStream -} - -type volumeServerVolumeEcShardReadClient struct { - grpc.ClientStream -} - -func (x *volumeServerVolumeEcShardReadClient) Recv() (*VolumeEcShardReadResponse, error) { - m := new(VolumeEcShardReadResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) { - out := new(VolumeEcBlobDeleteResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcBlobDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeEcShardsToVolume(ctx context.Context, in *VolumeEcShardsToVolumeRequest, opts ...grpc.CallOption) (*VolumeEcShardsToVolumeResponse, error) { - out := new(VolumeEcShardsToVolumeResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeTierMoveDatToRemote(ctx context.Context, in *VolumeTierMoveDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatToRemoteClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[4], "/volume_server_pb.VolumeServer/VolumeTierMoveDatToRemote", opts...) - if err != nil { - return nil, err - } - x := &volumeServerVolumeTierMoveDatToRemoteClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_VolumeTierMoveDatToRemoteClient interface { - Recv() (*VolumeTierMoveDatToRemoteResponse, error) - grpc.ClientStream -} - -type volumeServerVolumeTierMoveDatToRemoteClient struct { - grpc.ClientStream -} - -func (x *volumeServerVolumeTierMoveDatToRemoteClient) Recv() (*VolumeTierMoveDatToRemoteResponse, error) { - m := new(VolumeTierMoveDatToRemoteResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeTierMoveDatFromRemote(ctx context.Context, in *VolumeTierMoveDatFromRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatFromRemoteClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[5], "/volume_server_pb.VolumeServer/VolumeTierMoveDatFromRemote", opts...) - if err != nil { - return nil, err - } - x := &volumeServerVolumeTierMoveDatFromRemoteClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_VolumeTierMoveDatFromRemoteClient interface { - Recv() (*VolumeTierMoveDatFromRemoteResponse, error) - grpc.ClientStream -} - -type volumeServerVolumeTierMoveDatFromRemoteClient struct { - grpc.ClientStream -} - -func (x *volumeServerVolumeTierMoveDatFromRemoteClient) Recv() (*VolumeTierMoveDatFromRemoteResponse, error) { - m := new(VolumeTierMoveDatFromRemoteResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeServerStatus(ctx context.Context, in *VolumeServerStatusRequest, opts ...grpc.CallOption) (*VolumeServerStatusResponse, error) { - out := new(VolumeServerStatusResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeServerStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error) { - out := new(VolumeServerLeaveResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeServerLeave", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error) { - out := new(FetchAndWriteNeedleResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/FetchAndWriteNeedle", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) { - stream, err := c.cc.NewStream(ctx, &_VolumeServer_serviceDesc.Streams[6], "/volume_server_pb.VolumeServer/Query", opts...) - if err != nil { - return nil, err - } - x := &volumeServerQueryClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type VolumeServer_QueryClient interface { - Recv() (*QueriedStripe, error) - grpc.ClientStream -} - -type volumeServerQueryClient struct { - grpc.ClientStream -} - -func (x *volumeServerQueryClient) Recv() (*QueriedStripe, error) { - m := new(QueriedStripe) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *volumeServerClient) VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error) { - out := new(VolumeNeedleStatusResponse) - err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeNeedleStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// VolumeServerServer is the server API for VolumeServer service. -type VolumeServerServer interface { - //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. - BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) - VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) - VacuumVolumeCompact(context.Context, *VacuumVolumeCompactRequest) (*VacuumVolumeCompactResponse, error) - VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) - VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) - DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) - AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) - VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) - VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, VolumeServer_VolumeIncrementalCopyServer) error - VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) - VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) - VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) - VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) - VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) - VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) - VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) - // copy the .idx .dat files, and mount this volume - VolumeCopy(context.Context, *VolumeCopyRequest) (*VolumeCopyResponse, error) - ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) - CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error - ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) - WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) - VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error - VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) - // erasure coding - VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) - VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) - VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) - VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) - VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) - VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) - VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error - VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) - VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) - // tiered storage - VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, VolumeServer_VolumeTierMoveDatToRemoteServer) error - VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, VolumeServer_VolumeTierMoveDatFromRemoteServer) error - VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) - VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) - // remote storage - FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) - // query - Query(*QueryRequest, VolumeServer_QueryServer) error - VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) -} - -// UnimplementedVolumeServerServer can be embedded to have forward compatible implementations. -type UnimplementedVolumeServerServer struct { -} - -func (*UnimplementedVolumeServerServer) BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method BatchDelete not implemented") -} -func (*UnimplementedVolumeServerServer) VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCheck not implemented") -} -func (*UnimplementedVolumeServerServer) VacuumVolumeCompact(context.Context, *VacuumVolumeCompactRequest) (*VacuumVolumeCompactResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCompact not implemented") -} -func (*UnimplementedVolumeServerServer) VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCommit not implemented") -} -func (*UnimplementedVolumeServerServer) VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCleanup not implemented") -} -func (*UnimplementedVolumeServerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") -} -func (*UnimplementedVolumeServerServer) AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AllocateVolume not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeSyncStatus not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, VolumeServer_VolumeIncrementalCopyServer) error { - return status.Errorf(codes.Unimplemented, "method VolumeIncrementalCopy not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMount not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeUnmount not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeDelete not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkReadonly not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkWritable not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeConfigure not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeStatus not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeCopy(context.Context, *VolumeCopyRequest) (*VolumeCopyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeCopy not implemented") -} -func (*UnimplementedVolumeServerServer) ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReadVolumeFileStatus not implemented") -} -func (*UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error { - return status.Errorf(codes.Unimplemented, "method CopyFile not implemented") -} -func (*UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented") -} -func (*UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error { - return status.Errorf(codes.Unimplemented, "method VolumeTailSender not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeTailReceiver not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsCopy not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsDelete not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsMount not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error { - return status.Errorf(codes.Unimplemented, "method VolumeEcShardRead not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcBlobDelete not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, VolumeServer_VolumeTierMoveDatToRemoteServer) error { - return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, VolumeServer_VolumeTierMoveDatFromRemoteServer) error { - return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeServerStatus not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeServerLeave not implemented") -} -func (*UnimplementedVolumeServerServer) FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method FetchAndWriteNeedle not implemented") -} -func (*UnimplementedVolumeServerServer) Query(*QueryRequest, VolumeServer_QueryServer) error { - return status.Errorf(codes.Unimplemented, "method Query not implemented") -} -func (*UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented") -} - -func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) { - s.RegisterService(&_VolumeServer_serviceDesc, srv) -} - -func _VolumeServer_BatchDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(BatchDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).BatchDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/BatchDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).BatchDelete(ctx, req.(*BatchDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VacuumVolumeCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VacuumVolumeCheckRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VacuumVolumeCheck(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCheck", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VacuumVolumeCheck(ctx, req.(*VacuumVolumeCheckRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VacuumVolumeCompact_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VacuumVolumeCompactRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VacuumVolumeCompact(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCompact", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VacuumVolumeCompact(ctx, req.(*VacuumVolumeCompactRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VacuumVolumeCommit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VacuumVolumeCommitRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VacuumVolumeCommit(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCommit", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VacuumVolumeCommit(ctx, req.(*VacuumVolumeCommitRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VacuumVolumeCleanup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VacuumVolumeCleanupRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VacuumVolumeCleanup(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCleanup", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VacuumVolumeCleanup(ctx, req.(*VacuumVolumeCleanupRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_DeleteCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteCollectionRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).DeleteCollection(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/DeleteCollection", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).DeleteCollection(ctx, req.(*DeleteCollectionRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_AllocateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AllocateVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).AllocateVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/AllocateVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).AllocateVolume(ctx, req.(*AllocateVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeSyncStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeSyncStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeSyncStatus(ctx, req.(*VolumeSyncStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeIncrementalCopy_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(VolumeIncrementalCopyRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).VolumeIncrementalCopy(m, &volumeServerVolumeIncrementalCopyServer{stream}) -} - -type VolumeServer_VolumeIncrementalCopyServer interface { - Send(*VolumeIncrementalCopyResponse) error - grpc.ServerStream -} - -type volumeServerVolumeIncrementalCopyServer struct { - grpc.ServerStream -} - -func (x *volumeServerVolumeIncrementalCopyServer) Send(m *VolumeIncrementalCopyResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeMountRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeMount(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeMount", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeMount(ctx, req.(*VolumeMountRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeUnmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeUnmountRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeUnmount(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeUnmount", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeUnmount(ctx, req.(*VolumeUnmountRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeDelete(ctx, req.(*VolumeDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeMarkReadonly_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeMarkReadonlyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeMarkReadonly(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeMarkReadonly", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeMarkReadonly(ctx, req.(*VolumeMarkReadonlyRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeMarkWritable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeMarkWritableRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeMarkWritable(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeMarkWritable", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeMarkWritable(ctx, req.(*VolumeMarkWritableRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeConfigure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeConfigureRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeConfigure(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeConfigure", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeConfigure(ctx, req.(*VolumeConfigureRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeStatus(ctx, req.(*VolumeStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeCopyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeCopy(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeCopy", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeCopy(ctx, req.(*VolumeCopyRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_ReadVolumeFileStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReadVolumeFileStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).ReadVolumeFileStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/ReadVolumeFileStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).ReadVolumeFileStatus(ctx, req.(*ReadVolumeFileStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_CopyFile_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(CopyFileRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).CopyFile(m, &volumeServerCopyFileServer{stream}) -} - -type VolumeServer_CopyFileServer interface { - Send(*CopyFileResponse) error - grpc.ServerStream -} - -type volumeServerCopyFileServer struct { - grpc.ServerStream -} - -func (x *volumeServerCopyFileServer) Send(m *CopyFileResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_ReadNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReadNeedleBlobRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).ReadNeedleBlob(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/ReadNeedleBlob", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).ReadNeedleBlob(ctx, req.(*ReadNeedleBlobRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_WriteNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(WriteNeedleBlobRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).WriteNeedleBlob(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/WriteNeedleBlob", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).WriteNeedleBlob(ctx, req.(*WriteNeedleBlobRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeTailSender_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(VolumeTailSenderRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).VolumeTailSender(m, &volumeServerVolumeTailSenderServer{stream}) -} - -type VolumeServer_VolumeTailSenderServer interface { - Send(*VolumeTailSenderResponse) error - grpc.ServerStream -} - -type volumeServerVolumeTailSenderServer struct { - grpc.ServerStream -} - -func (x *volumeServerVolumeTailSenderServer) Send(m *VolumeTailSenderResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeTailReceiver_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeTailReceiverRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeTailReceiver(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeTailReceiver", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeTailReceiver(ctx, req.(*VolumeTailReceiverRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsGenerate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsGenerateRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, req.(*VolumeEcShardsGenerateRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsRebuild_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsRebuildRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsRebuild(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsRebuild(ctx, req.(*VolumeEcShardsRebuildRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsCopyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, req.(*VolumeEcShardsCopyRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, req.(*VolumeEcShardsDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsMountRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsMount", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, req.(*VolumeEcShardsMountRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsUnmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsUnmountRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, req.(*VolumeEcShardsUnmountRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardRead_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(VolumeEcShardReadRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).VolumeEcShardRead(m, &volumeServerVolumeEcShardReadServer{stream}) -} - -type VolumeServer_VolumeEcShardReadServer interface { - Send(*VolumeEcShardReadResponse) error - grpc.ServerStream -} - -type volumeServerVolumeEcShardReadServer struct { - grpc.ServerStream -} - -func (x *volumeServerVolumeEcShardReadServer) Send(m *VolumeEcShardReadResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeEcBlobDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcBlobDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcBlobDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcBlobDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcBlobDelete(ctx, req.(*VolumeEcBlobDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeEcShardsToVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeEcShardsToVolumeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeEcShardsToVolume(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeEcShardsToVolume(ctx, req.(*VolumeEcShardsToVolumeRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeTierMoveDatToRemote_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(VolumeTierMoveDatToRemoteRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).VolumeTierMoveDatToRemote(m, &volumeServerVolumeTierMoveDatToRemoteServer{stream}) -} - -type VolumeServer_VolumeTierMoveDatToRemoteServer interface { - Send(*VolumeTierMoveDatToRemoteResponse) error - grpc.ServerStream -} - -type volumeServerVolumeTierMoveDatToRemoteServer struct { - grpc.ServerStream -} - -func (x *volumeServerVolumeTierMoveDatToRemoteServer) Send(m *VolumeTierMoveDatToRemoteResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeTierMoveDatFromRemote_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(VolumeTierMoveDatFromRemoteRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).VolumeTierMoveDatFromRemote(m, &volumeServerVolumeTierMoveDatFromRemoteServer{stream}) -} - -type VolumeServer_VolumeTierMoveDatFromRemoteServer interface { - Send(*VolumeTierMoveDatFromRemoteResponse) error - grpc.ServerStream -} - -type volumeServerVolumeTierMoveDatFromRemoteServer struct { - grpc.ServerStream -} - -func (x *volumeServerVolumeTierMoveDatFromRemoteServer) Send(m *VolumeTierMoveDatFromRemoteResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeServerStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeServerStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeServerStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeServerStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeServerStatus(ctx, req.(*VolumeServerStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_VolumeServerLeave_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeServerLeaveRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeServerLeave(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeServerLeave", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeServerLeave(ctx, req.(*VolumeServerLeaveRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_FetchAndWriteNeedle_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(FetchAndWriteNeedleRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).FetchAndWriteNeedle(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/FetchAndWriteNeedle", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).FetchAndWriteNeedle(ctx, req.(*FetchAndWriteNeedleRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(QueryRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(VolumeServerServer).Query(m, &volumeServerQueryServer{stream}) -} - -type VolumeServer_QueryServer interface { - Send(*QueriedStripe) error - grpc.ServerStream -} - -type volumeServerQueryServer struct { - grpc.ServerStream -} - -func (x *volumeServerQueryServer) Send(m *QueriedStripe) error { - return x.ServerStream.SendMsg(m) -} - -func _VolumeServer_VolumeNeedleStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeNeedleStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeNeedleStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeNeedleStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeNeedleStatus(ctx, req.(*VolumeNeedleStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _VolumeServer_serviceDesc = grpc.ServiceDesc{ - ServiceName: "volume_server_pb.VolumeServer", - HandlerType: (*VolumeServerServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "BatchDelete", - Handler: _VolumeServer_BatchDelete_Handler, - }, - { - MethodName: "VacuumVolumeCheck", - Handler: _VolumeServer_VacuumVolumeCheck_Handler, - }, - { - MethodName: "VacuumVolumeCompact", - Handler: _VolumeServer_VacuumVolumeCompact_Handler, - }, - { - MethodName: "VacuumVolumeCommit", - Handler: _VolumeServer_VacuumVolumeCommit_Handler, - }, - { - MethodName: "VacuumVolumeCleanup", - Handler: _VolumeServer_VacuumVolumeCleanup_Handler, - }, - { - MethodName: "DeleteCollection", - Handler: _VolumeServer_DeleteCollection_Handler, - }, - { - MethodName: "AllocateVolume", - Handler: _VolumeServer_AllocateVolume_Handler, - }, - { - MethodName: "VolumeSyncStatus", - Handler: _VolumeServer_VolumeSyncStatus_Handler, - }, - { - MethodName: "VolumeMount", - Handler: _VolumeServer_VolumeMount_Handler, - }, - { - MethodName: "VolumeUnmount", - Handler: _VolumeServer_VolumeUnmount_Handler, - }, - { - MethodName: "VolumeDelete", - Handler: _VolumeServer_VolumeDelete_Handler, - }, - { - MethodName: "VolumeMarkReadonly", - Handler: _VolumeServer_VolumeMarkReadonly_Handler, - }, - { - MethodName: "VolumeMarkWritable", - Handler: _VolumeServer_VolumeMarkWritable_Handler, - }, - { - MethodName: "VolumeConfigure", - Handler: _VolumeServer_VolumeConfigure_Handler, - }, - { - MethodName: "VolumeStatus", - Handler: _VolumeServer_VolumeStatus_Handler, - }, - { - MethodName: "VolumeCopy", - Handler: _VolumeServer_VolumeCopy_Handler, - }, - { - MethodName: "ReadVolumeFileStatus", - Handler: _VolumeServer_ReadVolumeFileStatus_Handler, - }, - { - MethodName: "ReadNeedleBlob", - Handler: _VolumeServer_ReadNeedleBlob_Handler, - }, - { - MethodName: "WriteNeedleBlob", - Handler: _VolumeServer_WriteNeedleBlob_Handler, - }, - { - MethodName: "VolumeTailReceiver", - Handler: _VolumeServer_VolumeTailReceiver_Handler, - }, - { - MethodName: "VolumeEcShardsGenerate", - Handler: _VolumeServer_VolumeEcShardsGenerate_Handler, - }, - { - MethodName: "VolumeEcShardsRebuild", - Handler: _VolumeServer_VolumeEcShardsRebuild_Handler, - }, - { - MethodName: "VolumeEcShardsCopy", - Handler: _VolumeServer_VolumeEcShardsCopy_Handler, - }, - { - MethodName: "VolumeEcShardsDelete", - Handler: _VolumeServer_VolumeEcShardsDelete_Handler, - }, - { - MethodName: "VolumeEcShardsMount", - Handler: _VolumeServer_VolumeEcShardsMount_Handler, - }, - { - MethodName: "VolumeEcShardsUnmount", - Handler: _VolumeServer_VolumeEcShardsUnmount_Handler, - }, - { - MethodName: "VolumeEcBlobDelete", - Handler: _VolumeServer_VolumeEcBlobDelete_Handler, - }, - { - MethodName: "VolumeEcShardsToVolume", - Handler: _VolumeServer_VolumeEcShardsToVolume_Handler, - }, - { - MethodName: "VolumeServerStatus", - Handler: _VolumeServer_VolumeServerStatus_Handler, - }, - { - MethodName: "VolumeServerLeave", - Handler: _VolumeServer_VolumeServerLeave_Handler, - }, - { - MethodName: "FetchAndWriteNeedle", - Handler: _VolumeServer_FetchAndWriteNeedle_Handler, - }, - { - MethodName: "VolumeNeedleStatus", - Handler: _VolumeServer_VolumeNeedleStatus_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "VolumeIncrementalCopy", - Handler: _VolumeServer_VolumeIncrementalCopy_Handler, - ServerStreams: true, - }, - { - StreamName: "CopyFile", - Handler: _VolumeServer_CopyFile_Handler, - ServerStreams: true, - }, - { - StreamName: "VolumeTailSender", - Handler: _VolumeServer_VolumeTailSender_Handler, - ServerStreams: true, - }, - { - StreamName: "VolumeEcShardRead", - Handler: _VolumeServer_VolumeEcShardRead_Handler, - ServerStreams: true, - }, - { - StreamName: "VolumeTierMoveDatToRemote", - Handler: _VolumeServer_VolumeTierMoveDatToRemote_Handler, - ServerStreams: true, - }, - { - StreamName: "VolumeTierMoveDatFromRemote", - Handler: _VolumeServer_VolumeTierMoveDatFromRemote_Handler, - ServerStreams: true, - }, - { - StreamName: "Query", - Handler: _VolumeServer_Query_Handler, - ServerStreams: true, - }, - }, - Metadata: "volume_server.proto", -} diff --git a/weed/pb/volume_server_pb/volume_server_grpc.pb.go b/weed/pb/volume_server_pb/volume_server_grpc.pb.go new file mode 100644 index 000000000..e1b162457 --- /dev/null +++ b/weed/pb/volume_server_pb/volume_server_grpc.pb.go @@ -0,0 +1,1824 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package volume_server_pb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// VolumeServerClient is the client API for VolumeServer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type VolumeServerClient interface { + //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. + BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteResponse, error) + VacuumVolumeCheck(ctx context.Context, in *VacuumVolumeCheckRequest, opts ...grpc.CallOption) (*VacuumVolumeCheckResponse, error) + VacuumVolumeCompact(ctx context.Context, in *VacuumVolumeCompactRequest, opts ...grpc.CallOption) (VolumeServer_VacuumVolumeCompactClient, error) + VacuumVolumeCommit(ctx context.Context, in *VacuumVolumeCommitRequest, opts ...grpc.CallOption) (*VacuumVolumeCommitResponse, error) + VacuumVolumeCleanup(ctx context.Context, in *VacuumVolumeCleanupRequest, opts ...grpc.CallOption) (*VacuumVolumeCleanupResponse, error) + DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) + AllocateVolume(ctx context.Context, in *AllocateVolumeRequest, opts ...grpc.CallOption) (*AllocateVolumeResponse, error) + VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error) + VolumeIncrementalCopy(ctx context.Context, in *VolumeIncrementalCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeIncrementalCopyClient, error) + VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) + VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error) + VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) + VolumeMarkReadonly(ctx context.Context, in *VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*VolumeMarkReadonlyResponse, error) + VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error) + VolumeConfigure(ctx context.Context, in *VolumeConfigureRequest, opts ...grpc.CallOption) (*VolumeConfigureResponse, error) + VolumeStatus(ctx context.Context, in *VolumeStatusRequest, opts ...grpc.CallOption) (*VolumeStatusResponse, error) + // copy the .idx .dat files, and mount this volume + VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeCopyClient, error) + ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error) + CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) + ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error) + WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) + ReadAllNeedles(ctx context.Context, in *ReadAllNeedlesRequest, opts ...grpc.CallOption) (VolumeServer_ReadAllNeedlesClient, error) + VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error) + VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error) + // erasure coding + VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) + VolumeEcShardsRebuild(ctx context.Context, in *VolumeEcShardsRebuildRequest, opts ...grpc.CallOption) (*VolumeEcShardsRebuildResponse, error) + VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) + VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) + VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) + VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) + VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) + VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) + VolumeEcShardsToVolume(ctx context.Context, in *VolumeEcShardsToVolumeRequest, opts ...grpc.CallOption) (*VolumeEcShardsToVolumeResponse, error) + // tiered storage + VolumeTierMoveDatToRemote(ctx context.Context, in *VolumeTierMoveDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatToRemoteClient, error) + VolumeTierMoveDatFromRemote(ctx context.Context, in *VolumeTierMoveDatFromRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatFromRemoteClient, error) + VolumeServerStatus(ctx context.Context, in *VolumeServerStatusRequest, opts ...grpc.CallOption) (*VolumeServerStatusResponse, error) + VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error) + // remote storage + FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error) + // query + Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) + VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error) + Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) +} + +type volumeServerClient struct { + cc grpc.ClientConnInterface +} + +func NewVolumeServerClient(cc grpc.ClientConnInterface) VolumeServerClient { + return &volumeServerClient{cc} +} + +func (c *volumeServerClient) BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteResponse, error) { + out := new(BatchDeleteResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/BatchDelete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VacuumVolumeCheck(ctx context.Context, in *VacuumVolumeCheckRequest, opts ...grpc.CallOption) (*VacuumVolumeCheckResponse, error) { + out := new(VacuumVolumeCheckResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCheck", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VacuumVolumeCompact(ctx context.Context, in *VacuumVolumeCompactRequest, opts ...grpc.CallOption) (VolumeServer_VacuumVolumeCompactClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[0], "/volume_server_pb.VolumeServer/VacuumVolumeCompact", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVacuumVolumeCompactClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VacuumVolumeCompactClient interface { + Recv() (*VacuumVolumeCompactResponse, error) + grpc.ClientStream +} + +type volumeServerVacuumVolumeCompactClient struct { + grpc.ClientStream +} + +func (x *volumeServerVacuumVolumeCompactClient) Recv() (*VacuumVolumeCompactResponse, error) { + m := new(VacuumVolumeCompactResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VacuumVolumeCommit(ctx context.Context, in *VacuumVolumeCommitRequest, opts ...grpc.CallOption) (*VacuumVolumeCommitResponse, error) { + out := new(VacuumVolumeCommitResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCommit", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VacuumVolumeCleanup(ctx context.Context, in *VacuumVolumeCleanupRequest, opts ...grpc.CallOption) (*VacuumVolumeCleanupResponse, error) { + out := new(VacuumVolumeCleanupResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VacuumVolumeCleanup", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) { + out := new(DeleteCollectionResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/DeleteCollection", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) AllocateVolume(ctx context.Context, in *AllocateVolumeRequest, opts ...grpc.CallOption) (*AllocateVolumeResponse, error) { + out := new(AllocateVolumeResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/AllocateVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error) { + out := new(VolumeSyncStatusResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncStatus", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeIncrementalCopy(ctx context.Context, in *VolumeIncrementalCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeIncrementalCopyClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[1], "/volume_server_pb.VolumeServer/VolumeIncrementalCopy", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeIncrementalCopyClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeIncrementalCopyClient interface { + Recv() (*VolumeIncrementalCopyResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeIncrementalCopyClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeIncrementalCopyClient) Recv() (*VolumeIncrementalCopyResponse, error) { + m := new(VolumeIncrementalCopyResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) { + out := new(VolumeMountResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMount", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error) { + out := new(VolumeUnmountResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeUnmount", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) { + out := new(VolumeDeleteResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeDelete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeMarkReadonly(ctx context.Context, in *VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*VolumeMarkReadonlyResponse, error) { + out := new(VolumeMarkReadonlyResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMarkReadonly", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error) { + out := new(VolumeMarkWritableResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeMarkWritable", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeConfigure(ctx context.Context, in *VolumeConfigureRequest, opts ...grpc.CallOption) (*VolumeConfigureResponse, error) { + out := new(VolumeConfigureResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeConfigure", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeStatus(ctx context.Context, in *VolumeStatusRequest, opts ...grpc.CallOption) (*VolumeStatusResponse, error) { + out := new(VolumeStatusResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeStatus", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (VolumeServer_VolumeCopyClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[2], "/volume_server_pb.VolumeServer/VolumeCopy", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeCopyClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeCopyClient interface { + Recv() (*VolumeCopyResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeCopyClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeCopyClient) Recv() (*VolumeCopyResponse, error) { + m := new(VolumeCopyResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error) { + out := new(ReadVolumeFileStatusResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadVolumeFileStatus", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[3], "/volume_server_pb.VolumeServer/CopyFile", opts...) + if err != nil { + return nil, err + } + x := &volumeServerCopyFileClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_CopyFileClient interface { + Recv() (*CopyFileResponse, error) + grpc.ClientStream +} + +type volumeServerCopyFileClient struct { + grpc.ClientStream +} + +func (x *volumeServerCopyFileClient) Recv() (*CopyFileResponse, error) { + m := new(CopyFileResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error) { + out := new(ReadNeedleBlobResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadNeedleBlob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) { + out := new(WriteNeedleBlobResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/WriteNeedleBlob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) ReadAllNeedles(ctx context.Context, in *ReadAllNeedlesRequest, opts ...grpc.CallOption) (VolumeServer_ReadAllNeedlesClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[4], "/volume_server_pb.VolumeServer/ReadAllNeedles", opts...) + if err != nil { + return nil, err + } + x := &volumeServerReadAllNeedlesClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_ReadAllNeedlesClient interface { + Recv() (*ReadAllNeedlesResponse, error) + grpc.ClientStream +} + +type volumeServerReadAllNeedlesClient struct { + grpc.ClientStream +} + +func (x *volumeServerReadAllNeedlesClient) Recv() (*ReadAllNeedlesResponse, error) { + m := new(ReadAllNeedlesResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[5], "/volume_server_pb.VolumeServer/VolumeTailSender", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeTailSenderClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeTailSenderClient interface { + Recv() (*VolumeTailSenderResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeTailSenderClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeTailSenderClient) Recv() (*VolumeTailSenderResponse, error) { + m := new(VolumeTailSenderResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error) { + out := new(VolumeTailReceiverResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeTailReceiver", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) { + out := new(VolumeEcShardsGenerateResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsRebuild(ctx context.Context, in *VolumeEcShardsRebuildRequest, opts ...grpc.CallOption) (*VolumeEcShardsRebuildResponse, error) { + out := new(VolumeEcShardsRebuildResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) { + out := new(VolumeEcShardsCopyResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) { + out := new(VolumeEcShardsDeleteResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) { + out := new(VolumeEcShardsMountResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsMount", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) { + out := new(VolumeEcShardsUnmountResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[6], "/volume_server_pb.VolumeServer/VolumeEcShardRead", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeEcShardReadClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeEcShardReadClient interface { + Recv() (*VolumeEcShardReadResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeEcShardReadClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeEcShardReadClient) Recv() (*VolumeEcShardReadResponse, error) { + m := new(VolumeEcShardReadResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) { + out := new(VolumeEcBlobDeleteResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcBlobDelete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsToVolume(ctx context.Context, in *VolumeEcShardsToVolumeRequest, opts ...grpc.CallOption) (*VolumeEcShardsToVolumeResponse, error) { + out := new(VolumeEcShardsToVolumeResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeTierMoveDatToRemote(ctx context.Context, in *VolumeTierMoveDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatToRemoteClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[7], "/volume_server_pb.VolumeServer/VolumeTierMoveDatToRemote", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeTierMoveDatToRemoteClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeTierMoveDatToRemoteClient interface { + Recv() (*VolumeTierMoveDatToRemoteResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeTierMoveDatToRemoteClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeTierMoveDatToRemoteClient) Recv() (*VolumeTierMoveDatToRemoteResponse, error) { + m := new(VolumeTierMoveDatToRemoteResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeTierMoveDatFromRemote(ctx context.Context, in *VolumeTierMoveDatFromRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierMoveDatFromRemoteClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[8], "/volume_server_pb.VolumeServer/VolumeTierMoveDatFromRemote", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeTierMoveDatFromRemoteClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeTierMoveDatFromRemoteClient interface { + Recv() (*VolumeTierMoveDatFromRemoteResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeTierMoveDatFromRemoteClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeTierMoveDatFromRemoteClient) Recv() (*VolumeTierMoveDatFromRemoteResponse, error) { + m := new(VolumeTierMoveDatFromRemoteResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeServerStatus(ctx context.Context, in *VolumeServerStatusRequest, opts ...grpc.CallOption) (*VolumeServerStatusResponse, error) { + out := new(VolumeServerStatusResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeServerStatus", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error) { + out := new(VolumeServerLeaveResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeServerLeave", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error) { + out := new(FetchAndWriteNeedleResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/FetchAndWriteNeedle", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) { + stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[9], "/volume_server_pb.VolumeServer/Query", opts...) + if err != nil { + return nil, err + } + x := &volumeServerQueryClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_QueryClient interface { + Recv() (*QueriedStripe, error) + grpc.ClientStream +} + +type volumeServerQueryClient struct { + grpc.ClientStream +} + +func (x *volumeServerQueryClient) Recv() (*QueriedStripe, error) { + m := new(QueriedStripe) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *volumeServerClient) VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error) { + out := new(VolumeNeedleStatusResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeNeedleStatus", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { + out := new(PingResponse) + err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// VolumeServerServer is the server API for VolumeServer service. +// All implementations must embed UnimplementedVolumeServerServer +// for forward compatibility +type VolumeServerServer interface { + //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. + BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) + VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) + VacuumVolumeCompact(*VacuumVolumeCompactRequest, VolumeServer_VacuumVolumeCompactServer) error + VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) + VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) + DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) + AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) + VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) + VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, VolumeServer_VolumeIncrementalCopyServer) error + VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) + VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) + VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) + VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) + VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) + VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) + VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) + // copy the .idx .dat files, and mount this volume + VolumeCopy(*VolumeCopyRequest, VolumeServer_VolumeCopyServer) error + ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) + CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error + ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) + WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) + ReadAllNeedles(*ReadAllNeedlesRequest, VolumeServer_ReadAllNeedlesServer) error + VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error + VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) + // erasure coding + VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) + VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) + VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) + VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) + VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) + VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) + VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error + VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) + VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) + // tiered storage + VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, VolumeServer_VolumeTierMoveDatToRemoteServer) error + VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, VolumeServer_VolumeTierMoveDatFromRemoteServer) error + VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) + VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) + // remote storage + FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) + // query + Query(*QueryRequest, VolumeServer_QueryServer) error + VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) + Ping(context.Context, *PingRequest) (*PingResponse, error) + mustEmbedUnimplementedVolumeServerServer() +} + +// UnimplementedVolumeServerServer must be embedded to have forward compatible implementations. +type UnimplementedVolumeServerServer struct { +} + +func (UnimplementedVolumeServerServer) BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method BatchDelete not implemented") +} +func (UnimplementedVolumeServerServer) VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCheck not implemented") +} +func (UnimplementedVolumeServerServer) VacuumVolumeCompact(*VacuumVolumeCompactRequest, VolumeServer_VacuumVolumeCompactServer) error { + return status.Errorf(codes.Unimplemented, "method VacuumVolumeCompact not implemented") +} +func (UnimplementedVolumeServerServer) VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCommit not implemented") +} +func (UnimplementedVolumeServerServer) VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCleanup not implemented") +} +func (UnimplementedVolumeServerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") +} +func (UnimplementedVolumeServerServer) AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AllocateVolume not implemented") +} +func (UnimplementedVolumeServerServer) VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeSyncStatus not implemented") +} +func (UnimplementedVolumeServerServer) VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, VolumeServer_VolumeIncrementalCopyServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeIncrementalCopy not implemented") +} +func (UnimplementedVolumeServerServer) VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeMount not implemented") +} +func (UnimplementedVolumeServerServer) VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeUnmount not implemented") +} +func (UnimplementedVolumeServerServer) VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeDelete not implemented") +} +func (UnimplementedVolumeServerServer) VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkReadonly not implemented") +} +func (UnimplementedVolumeServerServer) VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkWritable not implemented") +} +func (UnimplementedVolumeServerServer) VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeConfigure not implemented") +} +func (UnimplementedVolumeServerServer) VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeStatus not implemented") +} +func (UnimplementedVolumeServerServer) VolumeCopy(*VolumeCopyRequest, VolumeServer_VolumeCopyServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeCopy not implemented") +} +func (UnimplementedVolumeServerServer) ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReadVolumeFileStatus not implemented") +} +func (UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error { + return status.Errorf(codes.Unimplemented, "method CopyFile not implemented") +} +func (UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented") +} +func (UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented") +} +func (UnimplementedVolumeServerServer) ReadAllNeedles(*ReadAllNeedlesRequest, VolumeServer_ReadAllNeedlesServer) error { + return status.Errorf(codes.Unimplemented, "method ReadAllNeedles not implemented") +} +func (UnimplementedVolumeServerServer) VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeTailSender not implemented") +} +func (UnimplementedVolumeServerServer) VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeTailReceiver not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsCopy not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsDelete not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsMount not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeEcShardRead not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcBlobDelete not implemented") +} +func (UnimplementedVolumeServerServer) VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented") +} +func (UnimplementedVolumeServerServer) VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, VolumeServer_VolumeTierMoveDatToRemoteServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented") +} +func (UnimplementedVolumeServerServer) VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, VolumeServer_VolumeTierMoveDatFromRemoteServer) error { + return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented") +} +func (UnimplementedVolumeServerServer) VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeServerStatus not implemented") +} +func (UnimplementedVolumeServerServer) VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeServerLeave not implemented") +} +func (UnimplementedVolumeServerServer) FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method FetchAndWriteNeedle not implemented") +} +func (UnimplementedVolumeServerServer) Query(*QueryRequest, VolumeServer_QueryServer) error { + return status.Errorf(codes.Unimplemented, "method Query not implemented") +} +func (UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented") +} +func (UnimplementedVolumeServerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {} + +// UnsafeVolumeServerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to VolumeServerServer will +// result in compilation errors. +type UnsafeVolumeServerServer interface { + mustEmbedUnimplementedVolumeServerServer() +} + +func RegisterVolumeServerServer(s grpc.ServiceRegistrar, srv VolumeServerServer) { + s.RegisterService(&VolumeServer_ServiceDesc, srv) +} + +func _VolumeServer_BatchDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BatchDeleteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).BatchDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/BatchDelete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).BatchDelete(ctx, req.(*BatchDeleteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VacuumVolumeCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VacuumVolumeCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VacuumVolumeCheck(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCheck", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VacuumVolumeCheck(ctx, req.(*VacuumVolumeCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VacuumVolumeCompact_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VacuumVolumeCompactRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VacuumVolumeCompact(m, &volumeServerVacuumVolumeCompactServer{stream}) +} + +type VolumeServer_VacuumVolumeCompactServer interface { + Send(*VacuumVolumeCompactResponse) error + grpc.ServerStream +} + +type volumeServerVacuumVolumeCompactServer struct { + grpc.ServerStream +} + +func (x *volumeServerVacuumVolumeCompactServer) Send(m *VacuumVolumeCompactResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VacuumVolumeCommit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VacuumVolumeCommitRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VacuumVolumeCommit(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCommit", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VacuumVolumeCommit(ctx, req.(*VacuumVolumeCommitRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VacuumVolumeCleanup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VacuumVolumeCleanupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VacuumVolumeCleanup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VacuumVolumeCleanup", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VacuumVolumeCleanup(ctx, req.(*VacuumVolumeCleanupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_DeleteCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteCollectionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).DeleteCollection(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/DeleteCollection", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).DeleteCollection(ctx, req.(*DeleteCollectionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_AllocateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AllocateVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).AllocateVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/AllocateVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).AllocateVolume(ctx, req.(*AllocateVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeSyncStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeSyncStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeSyncStatus(ctx, req.(*VolumeSyncStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeIncrementalCopy_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeIncrementalCopyRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeIncrementalCopy(m, &volumeServerVolumeIncrementalCopyServer{stream}) +} + +type VolumeServer_VolumeIncrementalCopyServer interface { + Send(*VolumeIncrementalCopyResponse) error + grpc.ServerStream +} + +type volumeServerVolumeIncrementalCopyServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeIncrementalCopyServer) Send(m *VolumeIncrementalCopyResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeMountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeMount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeMount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeMount(ctx, req.(*VolumeMountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeUnmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeUnmountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeUnmount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeUnmount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeUnmount(ctx, req.(*VolumeUnmountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeDeleteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeDelete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeDelete(ctx, req.(*VolumeDeleteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeMarkReadonly_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeMarkReadonlyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeMarkReadonly(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeMarkReadonly", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeMarkReadonly(ctx, req.(*VolumeMarkReadonlyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeMarkWritable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeMarkWritableRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeMarkWritable(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeMarkWritable", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeMarkWritable(ctx, req.(*VolumeMarkWritableRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeConfigure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeConfigureRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeConfigure(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeConfigure", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeConfigure(ctx, req.(*VolumeConfigureRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeStatus(ctx, req.(*VolumeStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeCopy_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeCopyRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeCopy(m, &volumeServerVolumeCopyServer{stream}) +} + +type VolumeServer_VolumeCopyServer interface { + Send(*VolumeCopyResponse) error + grpc.ServerStream +} + +type volumeServerVolumeCopyServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeCopyServer) Send(m *VolumeCopyResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_ReadVolumeFileStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReadVolumeFileStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).ReadVolumeFileStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/ReadVolumeFileStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).ReadVolumeFileStatus(ctx, req.(*ReadVolumeFileStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_CopyFile_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(CopyFileRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).CopyFile(m, &volumeServerCopyFileServer{stream}) +} + +type VolumeServer_CopyFileServer interface { + Send(*CopyFileResponse) error + grpc.ServerStream +} + +type volumeServerCopyFileServer struct { + grpc.ServerStream +} + +func (x *volumeServerCopyFileServer) Send(m *CopyFileResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_ReadNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReadNeedleBlobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).ReadNeedleBlob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/ReadNeedleBlob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).ReadNeedleBlob(ctx, req.(*ReadNeedleBlobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_WriteNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(WriteNeedleBlobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).WriteNeedleBlob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/WriteNeedleBlob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).WriteNeedleBlob(ctx, req.(*WriteNeedleBlobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_ReadAllNeedles_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadAllNeedlesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).ReadAllNeedles(m, &volumeServerReadAllNeedlesServer{stream}) +} + +type VolumeServer_ReadAllNeedlesServer interface { + Send(*ReadAllNeedlesResponse) error + grpc.ServerStream +} + +type volumeServerReadAllNeedlesServer struct { + grpc.ServerStream +} + +func (x *volumeServerReadAllNeedlesServer) Send(m *ReadAllNeedlesResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeTailSender_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeTailSenderRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeTailSender(m, &volumeServerVolumeTailSenderServer{stream}) +} + +type VolumeServer_VolumeTailSenderServer interface { + Send(*VolumeTailSenderResponse) error + grpc.ServerStream +} + +type volumeServerVolumeTailSenderServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeTailSenderServer) Send(m *VolumeTailSenderResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeTailReceiver_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeTailReceiverRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeTailReceiver(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeTailReceiver", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeTailReceiver(ctx, req.(*VolumeTailReceiverRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsGenerate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsGenerateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, req.(*VolumeEcShardsGenerateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsRebuild_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsRebuildRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsRebuild(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsRebuild(ctx, req.(*VolumeEcShardsRebuildRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsCopyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, req.(*VolumeEcShardsCopyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsDeleteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, req.(*VolumeEcShardsDeleteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsMountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsMount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, req.(*VolumeEcShardsMountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsUnmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsUnmountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, req.(*VolumeEcShardsUnmountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardRead_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeEcShardReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeEcShardRead(m, &volumeServerVolumeEcShardReadServer{stream}) +} + +type VolumeServer_VolumeEcShardReadServer interface { + Send(*VolumeEcShardReadResponse) error + grpc.ServerStream +} + +type volumeServerVolumeEcShardReadServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeEcShardReadServer) Send(m *VolumeEcShardReadResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeEcBlobDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcBlobDeleteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcBlobDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcBlobDelete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcBlobDelete(ctx, req.(*VolumeEcBlobDeleteRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsToVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsToVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsToVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsToVolume(ctx, req.(*VolumeEcShardsToVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeTierMoveDatToRemote_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeTierMoveDatToRemoteRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeTierMoveDatToRemote(m, &volumeServerVolumeTierMoveDatToRemoteServer{stream}) +} + +type VolumeServer_VolumeTierMoveDatToRemoteServer interface { + Send(*VolumeTierMoveDatToRemoteResponse) error + grpc.ServerStream +} + +type volumeServerVolumeTierMoveDatToRemoteServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeTierMoveDatToRemoteServer) Send(m *VolumeTierMoveDatToRemoteResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeTierMoveDatFromRemote_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeTierMoveDatFromRemoteRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeTierMoveDatFromRemote(m, &volumeServerVolumeTierMoveDatFromRemoteServer{stream}) +} + +type VolumeServer_VolumeTierMoveDatFromRemoteServer interface { + Send(*VolumeTierMoveDatFromRemoteResponse) error + grpc.ServerStream +} + +type volumeServerVolumeTierMoveDatFromRemoteServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeTierMoveDatFromRemoteServer) Send(m *VolumeTierMoveDatFromRemoteResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeServerStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeServerStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeServerStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeServerStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeServerStatus(ctx, req.(*VolumeServerStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeServerLeave_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeServerLeaveRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeServerLeave(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeServerLeave", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeServerLeave(ctx, req.(*VolumeServerLeaveRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_FetchAndWriteNeedle_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FetchAndWriteNeedleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).FetchAndWriteNeedle(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/FetchAndWriteNeedle", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).FetchAndWriteNeedle(ctx, req.(*FetchAndWriteNeedleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).Query(m, &volumeServerQueryServer{stream}) +} + +type VolumeServer_QueryServer interface { + Send(*QueriedStripe) error + grpc.ServerStream +} + +type volumeServerQueryServer struct { + grpc.ServerStream +} + +func (x *volumeServerQueryServer) Send(m *QueriedStripe) error { + return x.ServerStream.SendMsg(m) +} + +func _VolumeServer_VolumeNeedleStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeNeedleStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeNeedleStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeNeedleStatus", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeNeedleStatus(ctx, req.(*VolumeNeedleStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).Ping(ctx, req.(*PingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var VolumeServer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "volume_server_pb.VolumeServer", + HandlerType: (*VolumeServerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "BatchDelete", + Handler: _VolumeServer_BatchDelete_Handler, + }, + { + MethodName: "VacuumVolumeCheck", + Handler: _VolumeServer_VacuumVolumeCheck_Handler, + }, + { + MethodName: "VacuumVolumeCommit", + Handler: _VolumeServer_VacuumVolumeCommit_Handler, + }, + { + MethodName: "VacuumVolumeCleanup", + Handler: _VolumeServer_VacuumVolumeCleanup_Handler, + }, + { + MethodName: "DeleteCollection", + Handler: _VolumeServer_DeleteCollection_Handler, + }, + { + MethodName: "AllocateVolume", + Handler: _VolumeServer_AllocateVolume_Handler, + }, + { + MethodName: "VolumeSyncStatus", + Handler: _VolumeServer_VolumeSyncStatus_Handler, + }, + { + MethodName: "VolumeMount", + Handler: _VolumeServer_VolumeMount_Handler, + }, + { + MethodName: "VolumeUnmount", + Handler: _VolumeServer_VolumeUnmount_Handler, + }, + { + MethodName: "VolumeDelete", + Handler: _VolumeServer_VolumeDelete_Handler, + }, + { + MethodName: "VolumeMarkReadonly", + Handler: _VolumeServer_VolumeMarkReadonly_Handler, + }, + { + MethodName: "VolumeMarkWritable", + Handler: _VolumeServer_VolumeMarkWritable_Handler, + }, + { + MethodName: "VolumeConfigure", + Handler: _VolumeServer_VolumeConfigure_Handler, + }, + { + MethodName: "VolumeStatus", + Handler: _VolumeServer_VolumeStatus_Handler, + }, + { + MethodName: "ReadVolumeFileStatus", + Handler: _VolumeServer_ReadVolumeFileStatus_Handler, + }, + { + MethodName: "ReadNeedleBlob", + Handler: _VolumeServer_ReadNeedleBlob_Handler, + }, + { + MethodName: "WriteNeedleBlob", + Handler: _VolumeServer_WriteNeedleBlob_Handler, + }, + { + MethodName: "VolumeTailReceiver", + Handler: _VolumeServer_VolumeTailReceiver_Handler, + }, + { + MethodName: "VolumeEcShardsGenerate", + Handler: _VolumeServer_VolumeEcShardsGenerate_Handler, + }, + { + MethodName: "VolumeEcShardsRebuild", + Handler: _VolumeServer_VolumeEcShardsRebuild_Handler, + }, + { + MethodName: "VolumeEcShardsCopy", + Handler: _VolumeServer_VolumeEcShardsCopy_Handler, + }, + { + MethodName: "VolumeEcShardsDelete", + Handler: _VolumeServer_VolumeEcShardsDelete_Handler, + }, + { + MethodName: "VolumeEcShardsMount", + Handler: _VolumeServer_VolumeEcShardsMount_Handler, + }, + { + MethodName: "VolumeEcShardsUnmount", + Handler: _VolumeServer_VolumeEcShardsUnmount_Handler, + }, + { + MethodName: "VolumeEcBlobDelete", + Handler: _VolumeServer_VolumeEcBlobDelete_Handler, + }, + { + MethodName: "VolumeEcShardsToVolume", + Handler: _VolumeServer_VolumeEcShardsToVolume_Handler, + }, + { + MethodName: "VolumeServerStatus", + Handler: _VolumeServer_VolumeServerStatus_Handler, + }, + { + MethodName: "VolumeServerLeave", + Handler: _VolumeServer_VolumeServerLeave_Handler, + }, + { + MethodName: "FetchAndWriteNeedle", + Handler: _VolumeServer_FetchAndWriteNeedle_Handler, + }, + { + MethodName: "VolumeNeedleStatus", + Handler: _VolumeServer_VolumeNeedleStatus_Handler, + }, + { + MethodName: "Ping", + Handler: _VolumeServer_Ping_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "VacuumVolumeCompact", + Handler: _VolumeServer_VacuumVolumeCompact_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeIncrementalCopy", + Handler: _VolumeServer_VolumeIncrementalCopy_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeCopy", + Handler: _VolumeServer_VolumeCopy_Handler, + ServerStreams: true, + }, + { + StreamName: "CopyFile", + Handler: _VolumeServer_CopyFile_Handler, + ServerStreams: true, + }, + { + StreamName: "ReadAllNeedles", + Handler: _VolumeServer_ReadAllNeedles_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeTailSender", + Handler: _VolumeServer_VolumeTailSender_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeEcShardRead", + Handler: _VolumeServer_VolumeEcShardRead_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeTierMoveDatToRemote", + Handler: _VolumeServer_VolumeTierMoveDatToRemote_Handler, + ServerStreams: true, + }, + { + StreamName: "VolumeTierMoveDatFromRemote", + Handler: _VolumeServer_VolumeTierMoveDatFromRemote_Handler, + ServerStreams: true, + }, + { + StreamName: "Query", + Handler: _VolumeServer_Query_Handler, + ServerStreams: true, + }, + }, + Metadata: "volume_server.proto", +} diff --git a/weed/remote_storage/azure/azure_highlevel.go b/weed/remote_storage/azure/azure_highlevel.go index 9b735c2cb..a5cd4070b 100644 --- a/weed/remote_storage/azure/azure_highlevel.go +++ b/weed/remote_storage/azure/azure_highlevel.go @@ -41,7 +41,7 @@ func uploadReaderAtToBlockBlob(ctx context.Context, reader io.ReaderAt, readerSi if o.Progress != nil { body = pipeline.NewRequestBodyProgress(body, o.Progress) } - return blockBlobURL.Upload(ctx, body, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions) + return blockBlobURL.Upload(ctx, body, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions, o.ImmutabilityPolicyOptions) } var numBlocks = uint16(((readerSize - 1) / o.BlockSize) + 1) @@ -85,7 +85,7 @@ func uploadReaderAtToBlockBlob(ctx context.Context, reader io.ReaderAt, readerSi return nil, err } // All put blocks were successful, call Put Block List to finalize the blob - return blockBlobURL.CommitBlockList(ctx, blockIDList, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions) + return blockBlobURL.CommitBlockList(ctx, blockIDList, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions, o.ImmutabilityPolicyOptions) } // The UUID reserved variants. diff --git a/weed/remote_storage/azure/azure_storage_client.go b/weed/remote_storage/azure/azure_storage_client.go index 3982750f3..1a259a3e2 100644 --- a/weed/remote_storage/azure/azure_storage_client.go +++ b/weed/remote_storage/azure/azure_storage_client.go @@ -3,17 +3,17 @@ package azure import ( "context" "fmt" + "io" + "net/url" + "os" + "reflect" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" - "io" - "io/ioutil" - "net/url" - "os" - "reflect" ) func init() { @@ -115,7 +115,7 @@ func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocatio bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}) defer bodyStream.Close() - data, err = ioutil.ReadAll(bodyStream) + data, err = io.ReadAll(bodyStream) if err != nil { return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err) @@ -205,6 +205,7 @@ func (az *azureRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStor return } + func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { key := loc.Path[1:] containerURL := az.serviceURL.NewContainerURL(loc.Bucket) @@ -214,3 +215,38 @@ func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocat } return } + +func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) { + ctx := context.Background() + for containerMarker := (azblob.Marker{}); containerMarker.NotDone(); { + listContainer, err := az.serviceURL.ListContainersSegment(ctx, containerMarker, azblob.ListContainersSegmentOptions{}) + if err == nil { + for _, v := range listContainer.ContainerItems { + buckets = append(buckets, &remote_storage.Bucket{ + Name: v.Name, + CreatedAt: v.Properties.LastModified, + }) + } + } else { + return buckets, err + } + containerMarker = listContainer.NextMarker + } + return +} + +func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) { + containerURL := az.serviceURL.NewContainerURL(name) + if _, err = containerURL.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessNone); err != nil { + return fmt.Errorf("create bucket %s: %v", name, err) + } + return +} + +func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) { + containerURL := az.serviceURL.NewContainerURL(name) + if _, err = containerURL.Delete(context.Background(), azblob.ContainerAccessConditions{}); err != nil { + return fmt.Errorf("delete bucket %s: %v", name, err) + } + return +} diff --git a/weed/remote_storage/gcs/gcs_storage_client.go b/weed/remote_storage/gcs/gcs_storage_client.go index 683b90086..788d4b1e0 100644 --- a/weed/remote_storage/gcs/gcs_storage_client.go +++ b/weed/remote_storage/gcs/gcs_storage_client.go @@ -1,19 +1,20 @@ package gcs import ( - "cloud.google.com/go/storage" "context" "fmt" + "io" + "os" + "reflect" + + "cloud.google.com/go/storage" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" "google.golang.org/api/iterator" "google.golang.org/api/option" - "io" - "io/ioutil" - "os" - "reflect" ) func init() { @@ -41,6 +42,15 @@ func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage. } } + projectID := conf.GcsProjectId + if projectID == "" { + found := false + projectID, found = os.LookupEnv("GOOGLE_CLOUD_PROJECT") + if !found { + glog.Warningf("need to specific GOOGLE_CLOUD_PROJECT env variable") + } + } + googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials) c, err := storage.NewClient(context.Background(), option.WithCredentialsFile(googleApplicationCredentials)) @@ -49,12 +59,14 @@ func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage. } client.client = c + client.projectID = projectID return client, nil } type gcsRemoteStorageClient struct { - conf *remote_pb.RemoteConf - client *storage.Client + conf *remote_pb.RemoteConf + client *storage.Client + projectID string } var _ = remote_storage.RemoteStorageClient(&gcsRemoteStorageClient{}) @@ -98,7 +110,7 @@ func (gcs *gcsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation if readErr != nil { return nil, readErr } - data, err = ioutil.ReadAll(rangeReader) + data, err = io.ReadAll(rangeReader) if err != nil { return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err) @@ -169,7 +181,7 @@ func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStora if len(metadata) > 0 { _, err = gcs.client.Bucket(loc.Bucket).Object(key).Update(context.Background(), storage.ObjectAttrsToUpdate{ - Metadata: metadata, + Metadata: metadata, }) } else { // no way to delete the metadata yet @@ -184,3 +196,43 @@ func (gcs *gcsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocati } return } + +func (gcs *gcsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) { + if gcs.projectID == "" { + return nil, fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set") + } + iter := gcs.client.Buckets(context.Background(), gcs.projectID) + for { + b, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return buckets, err + } + buckets = append(buckets, &remote_storage.Bucket{ + Name: b.Name, + CreatedAt: b.Created, + }) + } + return +} + +func (gcs *gcsRemoteStorageClient) CreateBucket(name string) (err error) { + if gcs.projectID == "" { + return fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set") + } + err = gcs.client.Bucket(name).Create(context.Background(), gcs.projectID, &storage.BucketAttrs{}) + if err != nil { + return fmt.Errorf("create bucket %s: %v", name, err) + } + return +} + +func (gcs *gcsRemoteStorageClient) DeleteBucket(name string) (err error) { + err = gcs.client.Bucket(name).Delete(context.Background()) + if err != nil { + return fmt.Errorf("delete bucket %s: %v", name, err) + } + return +} diff --git a/weed/remote_storage/hdfs/hdfs_kerberos.go b/weed/remote_storage/hdfs/hdfs_kerberos.go deleted file mode 100644 index 50abc0ad5..000000000 --- a/weed/remote_storage/hdfs/hdfs_kerberos.go +++ /dev/null @@ -1,55 +0,0 @@ -package hdfs - -import ( - "fmt" - "os" - "os/user" - "strings" - - krb "github.com/jcmturner/gokrb5/v8/client" - "github.com/jcmturner/gokrb5/v8/config" - "github.com/jcmturner/gokrb5/v8/credentials" -) - -// copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go -func getKerberosClient() (*krb.Client, error) { - configPath := os.Getenv("KRB5_CONFIG") - if configPath == "" { - configPath = "/etc/krb5.conf" - } - - cfg, err := config.Load(configPath) - if err != nil { - return nil, err - } - - // Determine the ccache location from the environment, falling back to the - // default location. - ccachePath := os.Getenv("KRB5CCNAME") - if strings.Contains(ccachePath, ":") { - if strings.HasPrefix(ccachePath, "FILE:") { - ccachePath = strings.SplitN(ccachePath, ":", 2)[1] - } else { - return nil, fmt.Errorf("unusable ccache: %s", ccachePath) - } - } else if ccachePath == "" { - u, err := user.Current() - if err != nil { - return nil, err - } - - ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid) - } - - ccache, err := credentials.LoadCCache(ccachePath) - if err != nil { - return nil, err - } - - client, err := krb.NewFromCCache(ccache, cfg) - if err != nil { - return nil, err - } - - return client, nil -} diff --git a/weed/remote_storage/hdfs/hdfs_storage_client.go b/weed/remote_storage/hdfs/hdfs_storage_client.go deleted file mode 100644 index 5b4ce0b29..000000000 --- a/weed/remote_storage/hdfs/hdfs_storage_client.go +++ /dev/null @@ -1,178 +0,0 @@ -package hdfs - -import ( - "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" - "github.com/chrislusf/seaweedfs/weed/remote_storage" - "github.com/chrislusf/seaweedfs/weed/util" - "github.com/colinmarc/hdfs/v2" - "io" - "os" - "path" -) - -func init() { - remote_storage.RemoteStorageClientMakers["hdfs"] = new(hdfsRemoteStorageMaker) -} - -type hdfsRemoteStorageMaker struct{} - -func (s hdfsRemoteStorageMaker) HasBucket() bool { - return false -} - -func (s hdfsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { - client := &hdfsRemoteStorageClient{ - conf: conf, - } - - options := hdfs.ClientOptions{ - Addresses: conf.HdfsNamenodes, - UseDatanodeHostname: false, - } - - if conf.HdfsServicePrincipalName != "" { - var err error - options.KerberosClient, err = getKerberosClient() - if err != nil { - return nil, fmt.Errorf("get kerberos authentication: %s", err) - } - options.KerberosServicePrincipleName = conf.HdfsServicePrincipalName - - if conf.HdfsDataTransferProtection != "" { - options.DataTransferProtection = conf.HdfsDataTransferProtection - } - } else { - options.User = conf.HdfsUsername - } - - c, err := hdfs.NewClient(options) - if err != nil { - return nil, err - } - - client.client = c - return client, nil -} - -type hdfsRemoteStorageClient struct { - conf *remote_pb.RemoteConf - client *hdfs.Client -} - -var _ = remote_storage.RemoteStorageClient(&hdfsRemoteStorageClient{}) - -func (c *hdfsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { - - return remote_storage.TraverseBfs(func(parentDir util.FullPath, visitFn remote_storage.VisitFunc) error { - children, err := c.client.ReadDir(string(parentDir)) - if err != nil { - return err - } - for _, child := range children { - if err := visitFn(string(parentDir), child.Name(), child.IsDir(), &filer_pb.RemoteEntry{ - StorageName: c.conf.Name, - LastLocalSyncTsNs: 0, - RemoteETag: "", - RemoteMtime: child.ModTime().Unix(), - RemoteSize: child.Size(), - }); err != nil { - return nil - } - } - return nil - }, util.FullPath(loc.Path), visitFn) - -} -func (c *hdfsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) { - - f, err := c.client.Open(loc.Path) - if err != nil { - return - } - defer f.Close() - data = make([]byte, size) - _, err = f.ReadAt(data, offset) - - return - -} - -func (c *hdfsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) { - return c.client.MkdirAll(loc.Path, os.FileMode(entry.Attributes.FileMode)) -} - -func (c *hdfsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) { - return c.client.RemoveAll(loc.Path) -} - -func (c *hdfsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) { - - dirname := path.Dir(loc.Path) - - // ensure parent directory - if err = c.client.MkdirAll(dirname, 0755); err != nil { - return - } - - // remove existing file - info, err := c.client.Stat(loc.Path) - if err == nil { - err = c.client.Remove(loc.Path) - if err != nil { - return - } - } - - // create new file - out, err := c.client.Create(loc.Path) - if err != nil { - return - } - - cleanup := func() { - if removeErr := c.client.Remove(loc.Path); removeErr != nil { - glog.Errorf("clean up %s%s: %v", loc.Name, loc.Path, removeErr) - } - } - - if _, err = io.Copy(out, reader); err != nil { - cleanup() - return - } - - if err = out.Close(); err != nil { - cleanup() - return - } - - info, err = c.client.Stat(loc.Path) - if err != nil { - return - } - - return &filer_pb.RemoteEntry{ - RemoteMtime: info.ModTime().Unix(), - RemoteSize: info.Size(), - RemoteETag: "", - StorageName: c.conf.Name, - }, nil - -} - -func (c *hdfsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error { - if oldEntry.Attributes.FileMode != newEntry.Attributes.FileMode { - if err := c.client.Chmod(loc.Path, os.FileMode(newEntry.Attributes.FileMode)); err != nil { - return err - } - } - return nil -} -func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) { - if err = c.client.Remove(loc.Path); err != nil { - return fmt.Errorf("hdfs delete %s: %v", loc.Path, err) - } - return -} diff --git a/weed/remote_storage/remote_storage.go b/weed/remote_storage/remote_storage.go index af65ca677..e4a027199 100644 --- a/weed/remote_storage/remote_storage.go +++ b/weed/remote_storage/remote_storage.go @@ -4,16 +4,19 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/golang/protobuf/proto" "io" + "sort" "strings" "sync" + "time" ) +const slash = "/" + func ParseLocationName(remote string) (locationName string) { - if strings.HasSuffix(string(remote), "/") { - remote = remote[:len(remote)-1] - } - parts := strings.SplitN(string(remote), "/", 2) + remote = strings.TrimSuffix(remote, slash) + parts := strings.SplitN(remote, slash, 2) if len(parts) >= 1 { return parts[0] } @@ -22,45 +25,49 @@ func ParseLocationName(remote string) (locationName string) { func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { loc = &remote_pb.RemoteStorageLocation{} - if strings.HasSuffix(string(remote), "/") { - remote = remote[:len(remote)-1] - } - parts := strings.SplitN(string(remote), "/", 3) + remote = strings.TrimSuffix(remote, slash) + parts := strings.SplitN(remote, slash, 3) if len(parts) >= 1 { loc.Name = parts[0] } if len(parts) >= 2 { loc.Bucket = parts[1] } - loc.Path = string(remote[len(loc.Name)+1+len(loc.Bucket):]) + loc.Path = remote[len(loc.Name)+1+len(loc.Bucket):] if loc.Path == "" { - loc.Path = "/" + loc.Path = slash } return } func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) { loc = &remote_pb.RemoteStorageLocation{} - if strings.HasSuffix(string(remote), "/") { - remote = remote[:len(remote)-1] - } - parts := strings.SplitN(string(remote), "/", 2) + remote = strings.TrimSuffix(remote, slash) + parts := strings.SplitN(remote, slash, 2) if len(parts) >= 1 { loc.Name = parts[0] } - loc.Path = string(remote[len(loc.Name):]) + loc.Path = remote[len(loc.Name):] if loc.Path == "" { - loc.Path = "/" + loc.Path = slash } return } func FormatLocation(loc *remote_pb.RemoteStorageLocation) string { + if loc.Bucket == "" { + return fmt.Sprintf("%s%s", loc.Name, loc.Path) + } return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path) } type VisitFunc func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error +type Bucket struct { + Name string + CreatedAt time.Time +} + type RemoteStorageClient interface { Traverse(loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) @@ -69,6 +76,9 @@ type RemoteStorageClient interface { WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) + ListBuckets() ([]*Bucket, error) + CreateBucket(name string) (err error) + DeleteBucket(name string) (err error) } type RemoteStorageClientMaker interface { @@ -76,12 +86,37 @@ type RemoteStorageClientMaker interface { HasBucket() bool } +type CachedRemoteStorageClient struct { + *remote_pb.RemoteConf + RemoteStorageClient +} + var ( RemoteStorageClientMakers = make(map[string]RemoteStorageClientMaker) - remoteStorageClients = make(map[string]RemoteStorageClient) + remoteStorageClients = make(map[string]CachedRemoteStorageClient) remoteStorageClientsLock sync.Mutex ) +func GetAllRemoteStorageNames() string { + var storageNames []string + for k := range RemoteStorageClientMakers { + storageNames = append(storageNames, k) + } + sort.Strings(storageNames) + return strings.Join(storageNames, "|") +} + +func GetRemoteStorageNamesHasBucket() string { + var storageNames []string + for k, m := range RemoteStorageClientMakers { + if m.HasBucket() { + storageNames = append(storageNames, k) + } + } + sort.Strings(storageNames) + return strings.Join(storageNames, "|") +} + func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) { maker, found := RemoteStorageClientMakers[remoteConfType] if !found { @@ -107,8 +142,8 @@ func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, er defer remoteStorageClientsLock.Unlock() existingRemoteStorageClient, found := remoteStorageClients[remoteConf.Name] - if found { - return existingRemoteStorageClient, nil + if found && proto.Equal(existingRemoteStorageClient.RemoteConf, remoteConf) { + return existingRemoteStorageClient.RemoteStorageClient, nil } newRemoteStorageClient, err := makeRemoteStorageClient(remoteConf) @@ -116,7 +151,10 @@ func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, er return nil, fmt.Errorf("make remote storage client %s: %v", remoteConf.Name, err) } - remoteStorageClients[remoteConf.Name] = newRemoteStorageClient + remoteStorageClients[remoteConf.Name] = CachedRemoteStorageClient{ + RemoteConf: remoteConf, + RemoteStorageClient: newRemoteStorageClient, + } return newRemoteStorageClient, nil } diff --git a/weed/remote_storage/s3/aliyun.go b/weed/remote_storage/s3/aliyun.go index 567c74299..d6923aa6b 100644 --- a/weed/remote_storage/s3/aliyun.go +++ b/weed/remote_storage/s3/aliyun.go @@ -24,15 +24,16 @@ func (s AliyunRemoteStorageMaker) HasBucket() bool { func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: true, + conf: conf, } accessKey := util.Nvl(conf.AliyunAccessKey, os.Getenv("ALICLOUD_ACCESS_KEY_ID")) secretKey := util.Nvl(conf.AliyunSecretKey, os.Getenv("ALICLOUD_ACCESS_KEY_SECRET")) config := &aws.Config{ - Endpoint: aws.String(conf.AliyunEndpoint), - Region: aws.String(conf.AliyunRegion), - S3ForcePathStyle: aws.Bool(false), + Endpoint: aws.String(conf.AliyunEndpoint), + Region: aws.String(conf.AliyunRegion), + S3ForcePathStyle: aws.Bool(false), S3DisableContentMD5Validation: aws.Bool(true), } if accessKey != "" && secretKey != "" { diff --git a/weed/remote_storage/s3/backblaze.go b/weed/remote_storage/s3/backblaze.go index 914f0ca44..09a033f8c 100644 --- a/weed/remote_storage/s3/backblaze.go +++ b/weed/remote_storage/s3/backblaze.go @@ -22,12 +22,13 @@ func (s BackBlazeRemoteStorageMaker) HasBucket() bool { func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: false, + conf: conf, } config := &aws.Config{ - Endpoint: aws.String(conf.BackblazeEndpoint), - Region: aws.String("us-west-002"), - S3ForcePathStyle: aws.Bool(true), + Endpoint: aws.String(conf.BackblazeEndpoint), + Region: aws.String("us-west-002"), + S3ForcePathStyle: aws.Bool(true), S3DisableContentMD5Validation: aws.Bool(true), } if conf.BackblazeKeyId != "" && conf.BackblazeApplicationKey != "" { diff --git a/weed/remote_storage/s3/baidu.go b/weed/remote_storage/s3/baidu.go index dfcf32512..23bce409e 100644 --- a/weed/remote_storage/s3/baidu.go +++ b/weed/remote_storage/s3/baidu.go @@ -24,15 +24,16 @@ func (s BaiduRemoteStorageMaker) HasBucket() bool { func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: true, + conf: conf, } accessKey := util.Nvl(conf.BaiduAccessKey, os.Getenv("BDCLOUD_ACCESS_KEY")) secretKey := util.Nvl(conf.BaiduSecretKey, os.Getenv("BDCLOUD_SECRET_KEY")) config := &aws.Config{ - Endpoint: aws.String(conf.BaiduEndpoint), - Region: aws.String(conf.BaiduRegion), - S3ForcePathStyle: aws.Bool(true), + Endpoint: aws.String(conf.BaiduEndpoint), + Region: aws.String(conf.BaiduRegion), + S3ForcePathStyle: aws.Bool(true), S3DisableContentMD5Validation: aws.Bool(true), } if accessKey != "" && secretKey != "" { diff --git a/weed/remote_storage/s3/contabo.go b/weed/remote_storage/s3/contabo.go new file mode 100644 index 000000000..2e85422dc --- /dev/null +++ b/weed/remote_storage/s3/contabo.go @@ -0,0 +1,51 @@ +package s3 + +import ( + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + remote_storage.RemoteStorageClientMakers["contabo"] = new(ContaboRemoteStorageMaker) +} + +type ContaboRemoteStorageMaker struct{} + +func (s ContaboRemoteStorageMaker) HasBucket() bool { + return true +} + +func (s ContaboRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { + client := &s3RemoteStorageClient{ + supportTagging: true, + conf: conf, + } + accessKey := util.Nvl(conf.ContaboAccessKey, os.Getenv("ACCESS_KEY")) + secretKey := util.Nvl(conf.ContaboSecretKey, os.Getenv("SECRET_KEY")) + + config := &aws.Config{ + Endpoint: aws.String(conf.ContaboEndpoint), + Region: aws.String(conf.ContaboRegion), + S3ForcePathStyle: aws.Bool(true), + S3DisableContentMD5Validation: aws.Bool(true), + } + if accessKey != "" && secretKey != "" { + config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + + sess, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("create contabo session: %v", err) + } + sess.Handlers.Build.PushFront(skipSha256PayloadSigning) + client.conn = s3.New(sess) + return client, nil +} diff --git a/weed/remote_storage/s3/filebase.go b/weed/remote_storage/s3/filebase.go new file mode 100644 index 000000000..23787e0e2 --- /dev/null +++ b/weed/remote_storage/s3/filebase.go @@ -0,0 +1,52 @@ +package s3 + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" + "os" +) + +func init() { + remote_storage.RemoteStorageClientMakers["filebase"] = new(FilebaseRemoteStorageMaker) +} + +type FilebaseRemoteStorageMaker struct{} + +func (s FilebaseRemoteStorageMaker) HasBucket() bool { + return true +} + +func (s FilebaseRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { + client := &s3RemoteStorageClient{ + supportTagging: true, + conf: conf, + } + accessKey := util.Nvl(conf.FilebaseAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")) + secretKey := util.Nvl(conf.FilebaseSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")) + + config := &aws.Config{ + Endpoint: aws.String(conf.FilebaseEndpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + S3DisableContentMD5Validation: aws.Bool(true), + } + if accessKey != "" && secretKey != "" { + config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + + sess, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("create filebase session: %v", err) + } + sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler) + sess.Handlers.Build.PushFront(skipSha256PayloadSigning) + client.conn = s3.New(sess) + return client, nil +} diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go index 2756c91a8..b3c3cb3aa 100644 --- a/weed/remote_storage/s3/s3_storage_client.go +++ b/weed/remote_storage/s3/s3_storage_client.go @@ -4,7 +4,9 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -29,7 +31,8 @@ func (s s3RemoteStorageMaker) HasBucket() bool { func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: true, + conf: conf, } config := &aws.Config{ Region: aws.String(conf.S3Region), @@ -45,17 +48,24 @@ func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.R if err != nil { return nil, fmt.Errorf("create aws session: %v", err) } + if conf.S3V4Signature { + sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler) + } + sess.Handlers.Build.PushBack(func(r *request.Request) { + r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER) + }) sess.Handlers.Build.PushFront(skipSha256PayloadSigning) client.conn = s3.New(sess) return client, nil } type s3RemoteStorageClient struct { - conf *remote_pb.RemoteConf - conn s3iface.S3API + conf *remote_pb.RemoteConf + conn s3iface.S3API + supportTagging bool } -var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{}) +var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{supportTagging: true}) func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) { @@ -75,6 +85,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation } isLastPage := false for !isLastPage && err == nil { + var localErr error listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool { for _, content := range page.Contents { key := *content.Key @@ -86,6 +97,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation RemoteETag: *content.ETag, StorageName: s.conf.Name, }); err != nil { + localErr = err return false } } @@ -96,6 +108,9 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation if listErr != nil { err = fmt.Errorf("list %v: %v", remote, listErr) } + if localErr != nil { + err = fmt.Errorf("process %v: %v", remote, localErr) + } } return } @@ -145,11 +160,13 @@ func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, // process tagging tags := "" - for k, v := range entry.Extended { - if len(tags) > 0 { - tags = tags + "&" + if s.supportTagging { + for k, v := range entry.Extended { + if len(tags) > 0 { + tags = tags + "&" + } + tags = tags + k + "=" + string(v) } - tags = tags + k + "=" + string(v) } // Upload the file to S3. @@ -226,3 +243,45 @@ func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) }) return } + +func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) { + resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{}) + if err != nil { + return nil, fmt.Errorf("list buckets: %v", err) + } + for _, b := range resp.Buckets { + buckets = append(buckets, &remote_storage.Bucket{ + Name: *b.Name, + CreatedAt: *b.CreationDate, + }) + } + return +} + +func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) { + _, err = s.conn.CreateBucket(&s3.CreateBucketInput{ + ACL: nil, + Bucket: aws.String(name), + CreateBucketConfiguration: nil, + GrantFullControl: nil, + GrantRead: nil, + GrantReadACP: nil, + GrantWrite: nil, + GrantWriteACP: nil, + ObjectLockEnabledForBucket: nil, + }) + if err != nil { + return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err) + } + return +} + +func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) { + _, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(name), + }) + if err != nil { + return fmt.Errorf("delete bucket %s: %v", name, err) + } + return +} diff --git a/weed/remote_storage/s3/storj.go b/weed/remote_storage/s3/storj.go new file mode 100644 index 000000000..2de7ad357 --- /dev/null +++ b/weed/remote_storage/s3/storj.go @@ -0,0 +1,50 @@ +package s3 + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" + "os" +) + +func init() { + remote_storage.RemoteStorageClientMakers["storj"] = new(StorjRemoteStorageMaker) +} + +type StorjRemoteStorageMaker struct{} + +func (s StorjRemoteStorageMaker) HasBucket() bool { + return true +} + +func (s StorjRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { + client := &s3RemoteStorageClient{ + supportTagging: true, + conf: conf, + } + accessKey := util.Nvl(conf.StorjAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")) + secretKey := util.Nvl(conf.StorjSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")) + + config := &aws.Config{ + Endpoint: aws.String(conf.StorjEndpoint), + Region: aws.String("us-west-2"), + S3ForcePathStyle: aws.Bool(true), + S3DisableContentMD5Validation: aws.Bool(true), + } + if accessKey != "" && secretKey != "" { + config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + + sess, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("create storj session: %v", err) + } + sess.Handlers.Build.PushFront(skipSha256PayloadSigning) + client.conn = s3.New(sess) + return client, nil +} diff --git a/weed/remote_storage/s3/tencent.go b/weed/remote_storage/s3/tencent.go index 9df72a7e2..ab027a1f4 100644 --- a/weed/remote_storage/s3/tencent.go +++ b/weed/remote_storage/s3/tencent.go @@ -24,15 +24,16 @@ func (s TencentRemoteStorageMaker) HasBucket() bool { func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: true, + conf: conf, } accessKey := util.Nvl(conf.TencentSecretId, os.Getenv("COS_SECRETID")) secretKey := util.Nvl(conf.TencentSecretKey, os.Getenv("COS_SECRETKEY")) config := &aws.Config{ - Endpoint: aws.String(conf.TencentEndpoint), - Region: aws.String("us-west-2"), - S3ForcePathStyle: aws.Bool(true), + Endpoint: aws.String(conf.TencentEndpoint), + Region: aws.String("us-west-2"), + S3ForcePathStyle: aws.Bool(true), S3DisableContentMD5Validation: aws.Bool(true), } if accessKey != "" && secretKey != "" { diff --git a/weed/remote_storage/s3/wasabi.go b/weed/remote_storage/s3/wasabi.go index 29cdf7395..64dd0bbf2 100644 --- a/weed/remote_storage/s3/wasabi.go +++ b/weed/remote_storage/s3/wasabi.go @@ -24,7 +24,8 @@ func (s WasabiRemoteStorageMaker) HasBucket() bool { func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { client := &s3RemoteStorageClient{ - conf: conf, + supportTagging: true, + conf: conf, } accessKey := util.Nvl(conf.WasabiAccessKey) secretKey := util.Nvl(conf.WasabiSecretKey) diff --git a/weed/remote_storage/track_sync_offset.go b/weed/remote_storage/track_sync_offset.go new file mode 100644 index 000000000..25ac0d340 --- /dev/null +++ b/weed/remote_storage/track_sync_offset.go @@ -0,0 +1,73 @@ +package remote_storage + +import ( + "context" + "errors" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" +) + +const ( + SyncKeyPrefix = "remote.sync." +) + +func GetSyncOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, dir string) (lastOffsetTsNs int64, readErr error) { + + dirHash := uint32(util.HashStringToLong(dir)) + + readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + syncKey := []byte(SyncKeyPrefix + "____") + util.Uint32toBytes(syncKey[len(SyncKeyPrefix):len(SyncKeyPrefix)+4], dirHash) + + resp, err := client.KvGet(context.Background(), &filer_pb.KvGetRequest{Key: syncKey}) + if err != nil { + return err + } + + if len(resp.Error) != 0 { + return errors.New(resp.Error) + } + if len(resp.Value) < 8 { + return nil + } + + lastOffsetTsNs = int64(util.BytesToUint64(resp.Value)) + + return nil + }) + + return + +} + +func SetSyncOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, dir string, offsetTsNs int64) error { + + dirHash := uint32(util.HashStringToLong(dir)) + + return pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + + syncKey := []byte(SyncKeyPrefix + "____") + util.Uint32toBytes(syncKey[len(SyncKeyPrefix):len(SyncKeyPrefix)+4], dirHash) + + valueBuf := make([]byte, 8) + util.Uint64toBytes(valueBuf, uint64(offsetTsNs)) + + resp, err := client.KvPut(context.Background(), &filer_pb.KvPutRequest{ + Key: syncKey, + Value: valueBuf, + }) + if err != nil { + return err + } + + if len(resp.Error) != 0 { + return errors.New(resp.Error) + } + + return nil + + }) + +} diff --git a/weed/remote_storage/traverse_bfs.go b/weed/remote_storage/traverse_bfs.go index 4056f8715..a73a1d5fd 100644 --- a/weed/remote_storage/traverse_bfs.go +++ b/weed/remote_storage/traverse_bfs.go @@ -45,7 +45,7 @@ func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn } -func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) (error) { +func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) error { return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil { diff --git a/weed/replication/repl_util/replication_util.go b/weed/replication/repl_util/replication_util.go index 519a9a201..f135e6210 100644 --- a/weed/replication/repl_util/replication_util.go +++ b/weed/replication/repl_util/replication_util.go @@ -20,7 +20,7 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer var shouldRetry bool for _, fileUrl := range fileUrls { - shouldRetry, err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) { + shouldRetry, err = util.ReadUrlAsStream(fileUrl, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) { writeErr = writeFunc(data) }) if err != nil { diff --git a/weed/replication/replicator.go b/weed/replication/replicator.go index d7e609c68..eaab2c13e 100644 --- a/weed/replication/replicator.go +++ b/weed/replication/replicator.go @@ -82,8 +82,8 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p return r.sink.CreateEntry(key, message.NewEntry, message.Signatures) } -func ReadFilerSignature(grpcDialOption grpc.DialOption, filer string) (filerSignature int32, readErr error) { - if readErr = pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +func ReadFilerSignature(grpcDialOption grpc.DialOption, filer pb.ServerAddress) (filerSignature int32, readErr error) { + if readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}); err != nil { return fmt.Errorf("GetFilerConfiguration %s: %v", filer, err) } else { diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 3b944e5b7..3097ef439 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -109,7 +109,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [] // Azure Storage account's container. appendBlobURL := g.containerURL.NewAppendBlobURL(key) - _, err := appendBlobURL.Create(context.Background(), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) + _, err := appendBlobURL.Create(context.Background(), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}, azblob.ImmutabilityPolicyOptions{}) if err != nil { return err } diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index b5ea3e2cb..825d2af95 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -70,7 +70,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) var host string var auth security.EncodedJwt - if err := fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return util.Retry("assignVolume", func() error { request := &filer_pb.AssignVolumeRequest{ Count: 1, @@ -91,7 +91,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) return fmt.Errorf("assign volume failure %v: %v", request, resp.Error) } - fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth) + fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth) return nil }) @@ -107,7 +107,16 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) // fetch data as is, regardless whether it is encrypted or not - uploadResult, err, _ := operation.Upload(fileUrl, filename, false, resp.Body, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: fileUrl, + Filename: filename, + Cipher: false, + IsInputCompressed: "gzip" == header.Get("Content-Encoding"), + MimeType: header.Get("Content-Type"), + PairMap: nil, + Jwt: auth, + } + uploadResult, err, _ := operation.Upload(resp.Body, uploadOption) if err != nil { glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err) return "", fmt.Errorf("upload data: %v", err) @@ -122,9 +131,9 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) var _ = filer_pb.FilerClient(&FilerSink{}) -func (fs *FilerSink) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (fs *FilerSink) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) }, fs.grpcAddress, fs.grpcDialOption) diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index d42ca63a8..9471409fc 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -100,7 +100,7 @@ func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bo func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error { - return fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { dir, name := util.FullPath(key).DirAndName() @@ -156,7 +156,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // read existing entry var existingEntry *filer_pb.Entry - err = fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Directory: dir, @@ -199,7 +199,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // delete the chunks that are deleted from the source if deleteIncludeChunks { // remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks - existingEntry.Chunks = filer.DoMinusChunks(existingEntry.Chunks, deletedChunks) + existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.Chunks, deletedChunks) } // replicate the chunks that are new in the source @@ -208,10 +208,16 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent return true, fmt.Errorf("replicte %s chunks error: %v", key, err) } existingEntry.Chunks = append(existingEntry.Chunks, replicatedChunks...) + existingEntry.Attributes = newEntry.Attributes + existingEntry.Extended = newEntry.Extended + existingEntry.HardLinkId = newEntry.HardLinkId + existingEntry.HardLinkCounter = newEntry.HardLinkCounter + existingEntry.Content = newEntry.Content + existingEntry.RemoteEntry = newEntry.RemoteEntry } // save updated meta data - return true, fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return true, fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.UpdateEntryRequest{ Directory: newParentPath, diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go index 81cb87a18..a118324c2 100644 --- a/weed/replication/sink/s3sink/s3_sink.go +++ b/weed/replication/sink/s3sink/s3_sink.go @@ -26,6 +26,7 @@ type S3Sink struct { bucket string dir string endpoint string + acl string filerSource *source.FilerSource isIncremental bool } @@ -51,6 +52,7 @@ func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string glog.V(0).Infof("sink.s3.bucket: %v", configuration.GetString(prefix+"bucket")) glog.V(0).Infof("sink.s3.directory: %v", configuration.GetString(prefix+"directory")) glog.V(0).Infof("sink.s3.endpoint: %v", configuration.GetString(prefix+"endpoint")) + glog.V(0).Infof("sink.s3.acl: %v", configuration.GetString(prefix+"acl")) glog.V(0).Infof("sink.s3.is_incremental: %v", configuration.GetString(prefix+"is_incremental")) s3sink.isIncremental = configuration.GetBool(prefix + "is_incremental") return s3sink.initialize( @@ -60,6 +62,7 @@ func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string configuration.GetString(prefix+"bucket"), configuration.GetString(prefix+"directory"), configuration.GetString(prefix+"endpoint"), + configuration.GetString(prefix+"acl"), ) } @@ -67,16 +70,17 @@ func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) { s3sink.filerSource = s } -func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir, endpoint string) error { +func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir, endpoint, acl string) error { s3sink.region = region s3sink.bucket = bucket s3sink.dir = dir s3sink.endpoint = endpoint + s3sink.acl = acl config := &aws.Config{ - Region: aws.String(s3sink.region), - Endpoint: aws.String(s3sink.endpoint), - S3ForcePathStyle: aws.Bool(true), + Region: aws.String(s3sink.region), + Endpoint: aws.String(s3sink.endpoint), + S3ForcePathStyle: aws.Bool(true), S3DisableContentMD5Validation: aws.Bool(true), } if awsAccessKeyId != "" && awsSecretAccessKey != "" { diff --git a/weed/replication/sink/s3sink/s3_write.go b/weed/replication/sink/s3sink/s3_write.go index 3dde52616..7d8932fb0 100644 --- a/weed/replication/sink/s3sink/s3_write.go +++ b/weed/replication/sink/s3sink/s3_write.go @@ -39,6 +39,9 @@ func (s3sink *S3Sink) createMultipartUpload(key string, entry *filer_pb.Entry) ( Key: aws.String(key), ContentType: aws.String(entry.Attributes.Mime), } + if s3sink.acl != "" { + input.ACL = aws.String(s3sink.acl) + } result, err := s3sink.conn.CreateMultipartUpload(input) diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index 60c33463f..4108f3821 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -56,7 +56,7 @@ func (fs *FilerSource) LookupFileId(part string) (fileUrls []string, err error) vid := volumeId(part) - err = fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ VolumeIds: []string{vid}, @@ -118,9 +118,9 @@ func (fs *FilerSource) ReadPart(fileId string) (filename string, header http.Hea var _ = filer_pb.FilerClient(&FilerSource{}) -func (fs *FilerSource) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (fs *FilerSource) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) }, fs.grpcAddress, fs.grpcDialOption) diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go index 844aa023d..08133533f 100644 --- a/weed/replication/sub/notification_aws_sqs.go +++ b/weed/replication/sub/notification_aws_sqs.go @@ -41,7 +41,7 @@ func (k *AwsSqsInput) Initialize(configuration util.Configuration, prefix string func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) { config := &aws.Config{ - Region: aws.String(region), + Region: aws.String(region), S3DisableContentMD5Validation: aws.Bool(true), } if awsAccessKeyId != "" && awsSecretAccessKey != "" { diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index b16eec2e1..cb690e3ce 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + package sub import ( diff --git a/weed/replication/sub/notification_kafka.go b/weed/replication/sub/notification_kafka.go index 622a759ea..11bd2ffb4 100644 --- a/weed/replication/sub/notification_kafka.go +++ b/weed/replication/sub/notification_kafka.go @@ -3,7 +3,7 @@ package sub import ( "encoding/json" "fmt" - "io/ioutil" + "os" "sync" "time" @@ -119,7 +119,7 @@ type KafkaProgress struct { func loadProgress(offsetFile string) *KafkaProgress { progress := &KafkaProgress{} - data, err := ioutil.ReadFile(offsetFile) + data, err := os.ReadFile(offsetFile) if err != nil { glog.Warningf("failed to read kafka progress file: %s", offsetFile) return nil @@ -137,7 +137,7 @@ func (progress *KafkaProgress) saveProgress() error { if err != nil { return fmt.Errorf("failed to marshal progress: %v", err) } - err = ioutil.WriteFile(progress.offsetFile, data, 0640) + err = util.WriteFile(progress.offsetFile, data, 0640) if err != nil { return fmt.Errorf("failed to save progress to %s: %v", progress.offsetFile, err) } diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 44c3f7aa7..fb23d9ce9 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -2,17 +2,18 @@ package s3api import ( "fmt" + "net/http" + "os" + "strings" + "sync" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "io/ioutil" - "net/http" - "strings" ) type Action string @@ -22,8 +23,11 @@ type Iam interface { } type IdentityAccessManagement struct { - identities []*Identity - domain string + m sync.RWMutex + + identities []*Identity + isAuthEnabled bool + domain string } type Identity struct { @@ -37,6 +41,31 @@ type Credential struct { SecretKey string } +func (action Action) isAdmin() bool { + return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN) +} + +func (action Action) isOwner(bucket string) bool { + return string(action) == s3_constants.ACTION_ADMIN+":"+bucket +} + +func (action Action) overBucket(bucket string) bool { + return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*") +} + +func (action Action) getPermission() Permission { + switch act := strings.Split(string(action), ":")[0]; act { + case s3_constants.ACTION_ADMIN: + return Permission("FULL_CONTROL") + case s3_constants.ACTION_WRITE: + return Permission("WRITE") + case s3_constants.ACTION_READ: + return Permission("READ") + default: + return Permission("") + } +} + func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ domain: option.DomainName, @@ -55,26 +84,26 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) (err error) { var content []byte - err = pb.WithFilerClient(option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { content, err = filer.ReadInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile) return err }) if err != nil { return fmt.Errorf("read S3 config: %v", err) } - return iam.loadS3ApiConfigurationFromBytes(content) + return iam.LoadS3ApiConfigurationFromBytes(content) } func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error { - content, readErr := ioutil.ReadFile(fileName) + content, readErr := os.ReadFile(fileName) if readErr != nil { glog.Warningf("fail to read %s : %v", fileName, readErr) return fmt.Errorf("fail to read %s : %v", fileName, readErr) } - return iam.loadS3ApiConfigurationFromBytes(content) + return iam.LoadS3ApiConfigurationFromBytes(content) } -func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromBytes(content []byte) error { +func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []byte) error { s3ApiConfiguration := &iam_pb.S3ApiConfiguration{} if err := filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil { glog.Warningf("unmarshal error: %v", err) @@ -105,31 +134,39 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api } identities = append(identities, t) } - + iam.m.Lock() // atomically switch iam.identities = identities + if !iam.isAuthEnabled { // one-directional, no toggling + iam.isAuthEnabled = len(identities) > 0 + } + iam.m.Unlock() return nil } func (iam *IdentityAccessManagement) isEnabled() bool { - - return len(iam.identities) > 0 + return iam.isAuthEnabled } func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { + iam.m.RLock() + defer iam.m.RUnlock() for _, ident := range iam.identities { for _, cred := range ident.Credentials { + // println("checking", ident.Name, cred.AccessKey) if cred.AccessKey == accessKey { return ident, cred, true } } } + glog.V(1).Infof("could not find accessKey %s", accessKey) return nil, nil, false } func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) { - + iam.m.RLock() + defer iam.m.RUnlock() for _, ident := range iam.identities { if ident.Name == "anonymous" { return ident, true @@ -139,24 +176,26 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun } func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc { - - if !iam.isEnabled() { - return f - } - return func(w http.ResponseWriter, r *http.Request) { + if !iam.isEnabled() { + f(w, r) + return + } + identity, errCode := iam.authRequest(r, action) if errCode == s3err.ErrNone { if identity != nil && identity.Name != "" { - r.Header.Set(xhttp.AmzIdentityId, identity.Name) + r.Header.Set(s3_constants.AmzIdentityId, identity.Name) if identity.isAdmin() { - r.Header.Set(xhttp.AmzIsAdmin, "true") + r.Header.Set(s3_constants.AmzIsAdmin, "true") + } else if _, ok := r.Header[s3_constants.AmzIsAdmin]; ok { + r.Header.Del(s3_constants.AmzIsAdmin) } } f(w, r) return } - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) } } @@ -165,42 +204,53 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) var identity *Identity var s3Err s3err.ErrorCode var found bool + var authType string switch getRequestAuthType(r) { case authTypeStreamingSigned: return identity, s3err.ErrNone case authTypeUnknown: glog.V(3).Infof("unknown auth type") + r.Header.Set(s3_constants.AmzAuthType, "Unknown") return identity, s3err.ErrAccessDenied case authTypePresignedV2, authTypeSignedV2: glog.V(3).Infof("v2 auth type") identity, s3Err = iam.isReqAuthenticatedV2(r) + authType = "SigV2" case authTypeSigned, authTypePresigned: glog.V(3).Infof("v4 auth type") identity, s3Err = iam.reqSignatureV4Verify(r) + authType = "SigV4" case authTypePostPolicy: glog.V(3).Infof("post policy auth type") + r.Header.Set(s3_constants.AmzAuthType, "PostPolicy") return identity, s3err.ErrNone case authTypeJWT: glog.V(3).Infof("jwt auth type") + r.Header.Set(s3_constants.AmzAuthType, "Jwt") return identity, s3err.ErrNotImplemented case authTypeAnonymous: + authType = "Anonymous" identity, found = iam.lookupAnonymous() if !found { + r.Header.Set(s3_constants.AmzAuthType, authType) return identity, s3err.ErrAccessDenied } default: return identity, s3err.ErrNotImplemented } + if len(authType) > 0 { + r.Header.Set(s3_constants.AmzAuthType, authType) + } if s3Err != s3err.ErrNone { return identity, s3Err } glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action) - bucket, _ := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) - if !identity.canDo(action, bucket) { + if !identity.canDo(action, bucket, object) { return identity, s3err.ErrAccessDenied } @@ -212,33 +262,45 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err var identity *Identity var s3Err s3err.ErrorCode var found bool + var authType string switch getRequestAuthType(r) { case authTypeStreamingSigned: return identity, s3err.ErrNone case authTypeUnknown: glog.V(3).Infof("unknown auth type") + r.Header.Set(s3_constants.AmzAuthType, "Unknown") return identity, s3err.ErrAccessDenied case authTypePresignedV2, authTypeSignedV2: glog.V(3).Infof("v2 auth type") identity, s3Err = iam.isReqAuthenticatedV2(r) + authType = "SigV2" case authTypeSigned, authTypePresigned: glog.V(3).Infof("v4 auth type") identity, s3Err = iam.reqSignatureV4Verify(r) + authType = "SigV4" case authTypePostPolicy: glog.V(3).Infof("post policy auth type") + r.Header.Set(s3_constants.AmzAuthType, "PostPolicy") return identity, s3err.ErrNone case authTypeJWT: glog.V(3).Infof("jwt auth type") + r.Header.Set(s3_constants.AmzAuthType, "Jwt") return identity, s3err.ErrNotImplemented case authTypeAnonymous: + authType = "Anonymous" identity, found = iam.lookupAnonymous() if !found { + r.Header.Set(s3_constants.AmzAuthType, authType) return identity, s3err.ErrAccessDenied } default: return identity, s3err.ErrNotImplemented } + if len(authType) > 0 { + r.Header.Set(s3_constants.AmzAuthType, authType) + } + glog.V(3).Infof("auth error: %v", s3Err) if s3Err != s3err.ErrNone { return identity, s3Err @@ -246,7 +308,7 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err return identity, s3err.ErrNone } -func (identity *Identity) canDo(action Action, bucket string) bool { +func (identity *Identity) canDo(action Action, bucket string, objectKey string) bool { if identity.isAdmin() { return true } @@ -258,15 +320,17 @@ func (identity *Identity) canDo(action Action, bucket string) bool { if bucket == "" { return false } + target := string(action) + ":" + bucket + objectKey + adminTarget := s3_constants.ACTION_ADMIN + ":" + bucket + objectKey limitedByBucket := string(action) + ":" + bucket adminLimitedByBucket := s3_constants.ACTION_ADMIN + ":" + bucket for _, a := range identity.Actions { act := string(a) if strings.HasSuffix(act, "*") { - if strings.HasPrefix(limitedByBucket, act[:len(act)-1]) { + if strings.HasPrefix(target, act[:len(act)-1]) { return true } - if strings.HasPrefix(adminLimitedByBucket, act[:len(act)-1]) { + if strings.HasPrefix(adminTarget, act[:len(act)-1]) { return true } } else { diff --git a/weed/s3api/auth_credentials_subscribe.go b/weed/s3api/auth_credentials_subscribe.go index 05cce632a..f2bd94f56 100644 --- a/weed/s3api/auth_credentials_subscribe.go +++ b/weed/s3api/auth_credentials_subscribe.go @@ -5,10 +5,11 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/util" ) -func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) error { +func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) { processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error { @@ -22,18 +23,41 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la if message.NewParentPath != "" { dir = message.NewParentPath } - if dir == filer.IamConfigDirecotry && message.NewEntry.Name == filer.IamIdentityFile { - if err := s3a.iam.loadS3ApiConfigurationFromBytes(message.NewEntry.Content); err != nil { - return err - } - glog.V(0).Infof("updated %s/%s", filer.IamConfigDirecotry, filer.IamIdentityFile) - } + fileName := message.NewEntry.Name + content := message.NewEntry.Content + + _ = s3a.onIamConfigUpdate(dir, fileName, content) + _ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content) return nil } - return util.Retry("followIamChanges", func() error { - return pb.WithFilerClientFollowMetadata(s3a, clientName, prefix, lastTsNs, 0, processEventFn, true) + util.RetryForever("followIamChanges", func() error { + return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, prefix, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError) + }, func(err error) bool { + glog.V(0).Infof("iam follow metadata changes: %v", err) + return true }) +} +//reload iam config +func (s3a *S3ApiServer) onIamConfigUpdate(dir, filename string, content []byte) error { + if dir == filer.IamConfigDirecotry && filename == filer.IamIdentityFile { + if err := s3a.iam.LoadS3ApiConfigurationFromBytes(content); err != nil { + return err + } + glog.V(0).Infof("updated %s/%s", dir, filename) + } + return nil +} + +//reload circuit breaker config +func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, content []byte) error { + if dir == s3_constants.CircuitBreakerConfigDir && filename == s3_constants.CircuitBreakerConfigFile { + if err := s3a.cb.LoadS3ApiConfigurationFromBytes(content); err != nil { + return err + } + glog.V(0).Infof("updated %s/%s", dir, filename) + } + return nil } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index 0383ddbcd..4545d13bc 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -2,6 +2,7 @@ package s3api import ( . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/stretchr/testify/assert" "testing" "github.com/golang/protobuf/jsonpb" @@ -67,3 +68,61 @@ func TestIdentityListFileFormat(t *testing.T) { println(text) } + +func TestCanDo(t *testing.T) { + ident1 := &Identity{ + Name: "anything", + Actions: []Action{ + "Write:bucket1/a/b/c/*", + "Write:bucket1/a/b/other", + }, + } + // object specific + assert.Equal(t, true, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, false, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/other/some"), "action without *") + + // bucket specific + ident2 := &Identity{ + Name: "anything", + Actions: []Action{ + "Read:bucket1", + "Write:bucket1/*", + }, + } + assert.Equal(t, true, ident2.canDo(ACTION_READ, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, true, ident2.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, false, ident2.canDo(ACTION_LIST, "bucket1", "/a/b/c/d.txt")) + + // across buckets + ident3 := &Identity{ + Name: "anything", + Actions: []Action{ + "Read", + "Write", + }, + } + assert.Equal(t, true, ident3.canDo(ACTION_READ, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, true, ident3.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, false, ident3.canDo(ACTION_LIST, "bucket1", "/a/b/other/some")) + + // partial buckets + ident4 := &Identity{ + Name: "anything", + Actions: []Action{ + "Read:special_*", + }, + } + assert.Equal(t, true, ident4.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt")) + assert.Equal(t, false, ident4.canDo(ACTION_READ, "bucket1", "/a/b/c/d.txt")) + + // admin buckets + ident5 := &Identity{ + Name: "anything", + Actions: []Action{ + "Admin:special_*", + }, + } + assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt")) + assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt")) + +} diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 0df26e6fc..a49caad06 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -23,8 +23,7 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "io/ioutil" + "io" "net/http" "net/url" "regexp" @@ -33,6 +32,8 @@ import ( "strings" "time" "unicode/utf8" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" ) func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Identity, s3err.ErrorCode) { @@ -135,9 +136,9 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r // Get hashed Payload if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { - buf, _ := ioutil.ReadAll(r.Body) - r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) - b, _ := ioutil.ReadAll(bytes.NewBuffer(buf)) + buf, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewBuffer(buf)) + b, _ := io.ReadAll(bytes.NewBuffer(buf)) if len(b) != 0 { bodyHash := sha256.Sum256(b) hashedPayload = hex.EncodeToString(bodyHash[:]) @@ -433,7 +434,7 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s } } - /// Verify finally if signature is same. + // / Verify finally if signature is same. // Get canonical request. presignedCanonicalReq := getCanonicalRequest(extractedSignedHeaders, hashedPayload, encodedQuery, req.URL.Path, req.Method) diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index b47cd5f2d..a58551187 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -8,9 +8,7 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "io" - "io/ioutil" "net/http" "net/url" "sort" @@ -19,6 +17,8 @@ import ( "testing" "time" "unicode/utf8" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" ) // TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection. @@ -86,7 +86,7 @@ func TestIsReqAuthenticated(t *testing.T) { // Validates all testcases. for i, testCase := range testCases { if _, s3Error := iam.reqSignatureV4Verify(testCase.req); s3Error != testCase.s3Error { - ioutil.ReadAll(testCase.req.Body) + io.ReadAll(testCase.req.Body) t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d", i, testCase.s3Error, s3Error) } } @@ -167,7 +167,7 @@ func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeek case body == nil: hashedPayload = getSHA256Hash([]byte{}) default: - payloadBytes, err := ioutil.ReadAll(body) + payloadBytes, err := io.ReadAll(body) if err != nil { return nil, err } diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index ec26f693a..2678f312f 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -24,6 +24,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "hash" "io" @@ -90,8 +91,8 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID } - bucket, _ := getBucketAndObject(r) - if !identity.canDo("Write", bucket) { + bucket, object := s3_constants.GetBucketAndObject(r) + if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) { errCode = s3err.ErrAccessDenied return } diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 2b6707f2e..32b93307a 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1,17 +1,20 @@ package s3api import ( + "encoding/hex" "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "golang.org/x/exp/slices" + "math" "path/filepath" + "sort" "strconv" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" - "github.com/google/uuid" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -27,8 +30,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp glog.V(2).Infof("createMultipartUpload input %v", input) - uploadId, _ := uuid.NewRandom() - uploadIdString := uploadId.String() + uploadIdString := s3a.generateUploadID(*input.Key) if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { if entry.Extended == nil { @@ -38,6 +40,9 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp for k, v := range input.Metadata { entry.Extended[k] = []byte(*v) } + if input.ContentType != nil { + entry.Attributes.Mime = *input.ContentType + } }); err != nil { glog.Errorf("NewMultipartUpload error: %v", err) return nil, s3err.ErrInternalError @@ -59,13 +64,18 @@ type CompleteMultipartUploadResult struct { s3.CompleteMultipartUploadOutput } -func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { +func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("completeMultipartUpload input %v", input) + completedParts := parts.Parts + slices.SortFunc(completedParts, func(a, b CompletedPart) bool { + return a.PartNumber < b.PartNumber + }) + uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId - entries, _, err := s3a.list(uploadDirectory, "", "", false, 0) + entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) if err != nil || len(entries) == 0 { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) return nil, s3err.ErrNoSuchUpload @@ -77,11 +87,22 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return nil, s3err.ErrNoSuchUpload } + mime := pentry.Attributes.Mime + var finalParts []*filer_pb.FileChunk var offset int64 for _, entry := range entries { if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { + partETag, found := findByPartNumber(entry.Name, completedParts) + if !found { + continue + } + entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) + if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { + glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) + return nil, s3err.ErrInvalidPart + } for _, chunk := range entry.Chunks { p := &filer_pb.FileChunk{ FileId: chunk.GetFileIdString(), @@ -121,6 +142,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa entry.Extended[k] = v } } + if pentry.Attributes.Mime != "" { + entry.Attributes.Mime = pentry.Attributes.Mime + } else if mime != "" { + entry.Attributes.Mime = mime + } }) if err != nil { @@ -130,7 +156,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa output = &CompleteMultipartUploadResult{ CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{ - Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)), + Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlPathEscape(dirName), urlPathEscape(entryName))), Bucket: input.Bucket, ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""), Key: objectKey(input.Key), @@ -144,6 +170,31 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } +func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) { + partNumber, formatErr := strconv.Atoi(fileName[:4]) + if formatErr != nil { + return + } + x := sort.Search(len(parts), func(i int) bool { + return parts[i].PartNumber >= partNumber + }) + if x >= len(parts) { + return + } + if parts[x].PartNumber != partNumber { + return + } + y := 0 + for i, part := range parts[x:] { + if part.PartNumber == partNumber { + y = i + } else { + break + } + } + return parts[x+y].ETag, true +} + func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { glog.V(2).Infof("abortMultipartUpload input %v", input) @@ -195,13 +246,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput Prefix: input.Prefix, } - entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", *input.UploadIdMarker, false, uint32(*input.MaxUploads)) + entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", *input.UploadIdMarker, false, math.MaxInt32) if err != nil { glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err) return } - output.IsTruncated = aws.Bool(!isLast) + uploadsCount := int64(0) for _, entry := range entries { if entry.Extended != nil { key := string(entry.Extended["key"]) @@ -215,9 +266,12 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput Key: objectKey(aws.String(key)), UploadId: aws.String(entry.Name), }) - if !isLast { - output.NextUploadIdMarker = aws.String(entry.Name) - } + uploadsCount += 1 + } + if uploadsCount >= *input.MaxUploads { + output.IsTruncated = aws.Bool(true) + output.NextUploadIdMarker = aws.String(entry.Name) + break } } @@ -259,6 +313,9 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP return nil, s3err.ErrNoSuchUpload } + // Note: The upload directory is sort of a marker of the existence of an multipart upload request. + // So can not just delete empty upload folders. + output.IsTruncated = aws.Bool(!isLast) for _, entry := range entries { diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go index 9e1d2307b..fe2b9c0ce 100644 --- a/weed/s3api/filer_multipart_test.go +++ b/weed/s3api/filer_multipart_test.go @@ -4,6 +4,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/stretchr/testify/assert" "testing" "time" ) @@ -48,3 +49,89 @@ func TestListPartsResult(t *testing.T) { } } + +func Test_findByPartNumber(t *testing.T) { + type args struct { + fileName string + parts []CompletedPart + } + + parts := []CompletedPart{ + CompletedPart{ + ETag: "xxx", + PartNumber: 1, + }, + CompletedPart{ + ETag: "lll", + PartNumber: 1, + }, + CompletedPart{ + ETag: "yyy", + PartNumber: 3, + }, + CompletedPart{ + ETag: "zzz", + PartNumber: 5, + }, + } + + tests := []struct { + name string + args args + wantEtag string + wantFound bool + }{ + { + "first", + args{ + "0001.part", + parts, + }, + "lll", + true, + }, + { + "second", + args{ + "0002.part", + parts, + }, + "", + false, + }, + { + "third", + args{ + "0003.part", + parts, + }, + "yyy", + true, + }, + { + "fourth", + args{ + "0004.part", + parts, + }, + "", + false, + }, + { + "fifth", + args{ + "0005.part", + parts, + }, + "zzz", + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotEtag, gotFound := findByPartNumber(tt.args.fileName, tt.args.parts) + assert.Equalf(t, tt.wantEtag, gotEtag, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) + assert.Equalf(t, tt.wantFound, gotFound, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) + }) + } +} diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index 888003e45..dbd667339 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -41,7 +41,7 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error { - return s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive) if err != nil { @@ -55,10 +55,11 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error { request := &filer_pb.DeleteEntryRequest{ - Directory: parentDirectoryPath, - Name: entryName, - IsDeleteData: isDeleteData, - IsRecursive: isRecursive, + Directory: parentDirectoryPath, + Name: entryName, + IsDeleteData: isDeleteData, + IsRecursive: isRecursive, + IgnoreRecursiveError: true, } glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request) diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index 75d3b37d0..18d4d69c5 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -1,19 +1,19 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "strings" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" ) const ( - S3TAG_PREFIX = xhttp.AmzObjectTagging + "-" + S3TAG_PREFIX = s3_constants.AmzObjectTagging + "-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { - err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ Directory: parentDirectoryPath, @@ -35,7 +35,7 @@ func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (t func (s3a *S3ApiServer) setTags(parentDirectoryPath string, entryName string, tags map[string]string) (err error) { - return s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ Directory: parentDirectoryPath, @@ -71,7 +71,7 @@ func (s3a *S3ApiServer) setTags(parentDirectoryPath string, entryName string, ta func (s3a *S3ApiServer) rmTags(parentDirectoryPath string, entryName string) (err error) { - return s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ Directory: parentDirectoryPath, diff --git a/weed/s3api/http/header.go b/weed/s3api/http/header.go deleted file mode 100644 index 6614b0af0..000000000 --- a/weed/s3api/http/header.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package http - -// Standard S3 HTTP request constants -const ( - // S3 storage class - AmzStorageClass = "x-amz-storage-class" - - // S3 user-defined metadata - AmzUserMetaPrefix = "X-Amz-Meta-" - - // S3 object tagging - AmzObjectTagging = "X-Amz-Tagging" - AmzTagCount = "x-amz-tagging-count" -) - -// Non-Standard S3 HTTP request constants -const ( - AmzIdentityId = "s3-identity-id" - AmzIsAdmin = "s3-is-admin" // only set to http request header as a context -) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go new file mode 100644 index 000000000..cd725d435 --- /dev/null +++ b/weed/s3api/s3_constants/header.go @@ -0,0 +1,66 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3_constants + +import ( + "github.com/gorilla/mux" + "net/http" + "strings" +) + +// Standard S3 HTTP request constants +const ( + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 user-defined metadata + AmzUserMetaPrefix = "X-Amz-Meta-" + AmzUserMetaDirective = "X-Amz-Metadata-Directive" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzObjectTaggingPrefix = "X-Amz-Tagging-" + AmzObjectTaggingDirective = "X-Amz-Tagging-Directive" + AmzTagCount = "x-amz-tagging-count" +) + +// Non-Standard S3 HTTP request constants +const ( + AmzIdentityId = "s3-identity-id" + AmzAuthType = "s3-auth-type" + AmzIsAdmin = "s3-is-admin" // only set to http request header as a context +) + +func GetBucketAndObject(r *http.Request) (bucket, object string) { + vars := mux.Vars(r) + bucket = vars["bucket"] + object = vars["object"] + if !strings.HasPrefix(object, "/") { + object = "/" + object + } + + return +} + +var PassThroughHeaders = map[string]string{ + "response-cache-control": "Cache-Control", + "response-content-disposition": "Content-Disposition", + "response-content-encoding": "Content-Encoding", + "response-content-language": "Content-Language", + "response-content-type": "Content-Type", + "response-expires": "Expires", +} diff --git a/weed/s3api/s3_constants/s3_actions.go b/weed/s3api/s3_constants/s3_actions.go index 4e484ac98..0fbf134e3 100644 --- a/weed/s3api/s3_constants/s3_actions.go +++ b/weed/s3api/s3_constants/s3_actions.go @@ -6,4 +6,6 @@ const ( ACTION_ADMIN = "Admin" ACTION_TAGGING = "Tagging" ACTION_LIST = "List" + + SeaweedStorageDestinationHeader = "x-seaweedfs-destination" ) diff --git a/weed/s3api/s3_constants/s3_config.go b/weed/s3api/s3_constants/s3_config.go new file mode 100644 index 000000000..0fa5b26f4 --- /dev/null +++ b/weed/s3api/s3_constants/s3_config.go @@ -0,0 +1,18 @@ +package s3_constants + +import ( + "strings" +) + +var ( + CircuitBreakerConfigDir = "/etc/s3" + CircuitBreakerConfigFile = "circuit_breaker.json" + AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN} + LimitTypeCount = "Count" + LimitTypeBytes = "MB" + Separator = ":" +) + +func Concat(elements ...string) string { + return strings.Join(elements, Separator) +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 8beb954aa..f70e46b92 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -3,13 +3,16 @@ package s3api import ( "context" "encoding/xml" + "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "math" "net/http" "time" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/aws/aws-sdk-go/aws" @@ -27,12 +30,14 @@ type ListAllMyBucketsResult struct { func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { + glog.V(3).Infof("ListBucketsHandler") + var identity *Identity var s3Err s3err.ErrorCode if s3a.iam.isEnabled() { identity, s3Err = s3a.iam.authUser(r) if s3Err != s3err.ErrNone { - s3err.WriteErrorResponse(w, s3Err, r) + s3err.WriteErrorResponse(w, r, s3Err) return } } @@ -42,16 +47,16 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - identityId := r.Header.Get(xhttp.AmzIdentityId) + identityId := r.Header.Get(s3_constants.AmzIdentityId) var buckets []*s3.Bucket for _, entry := range entries { if entry.IsDirectory { - if identity != nil && !identity.canDo(s3_constants.ACTION_LIST, entry.Name) { + if identity != nil && !identity.canDo(s3_constants.ACTION_LIST, entry.Name, "") { continue } buckets = append(buckets, &s3.Bucket{ @@ -69,16 +74,17 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques Buckets: buckets, } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutBucketHandler %s", bucket) // avoid duplicated buckets errCode := s3err.ErrNone - if err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { if resp, err := client.CollectionList(context.Background(), &filer_pb.CollectionListRequest{ IncludeEcVolumes: true, IncludeNormalVolumes: true, @@ -95,46 +101,63 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } return nil }); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist { errCode = s3err.ErrBucketAlreadyExists } if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } + if s3a.iam.isEnabled() { + if _, errCode = s3a.iam.authRequest(r, s3_constants.ACTION_ADMIN); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + } + fn := func(entry *filer_pb.Entry) { - if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" { + if identityId := r.Header.Get(s3_constants.AmzIdentityId); identityId != "" { if entry.Extended == nil { entry.Extended = make(map[string][]byte) } - entry.Extended[xhttp.AmzIdentityId] = []byte(identityId) + entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) } } // create the folder for bucket, but lazily create actual collection if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil { glog.Errorf("PutBucketHandler mkdir: %v", err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - - writeSuccessResponseEmpty(w) + w.Header().Set("Location", "/"+bucket) + writeSuccessResponseEmpty(w, r) } func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("DeleteBucketHandler %s", bucket) if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { - s3err.WriteErrorResponse(w, err, r) + s3err.WriteErrorResponse(w, r, err) return } - err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + if !s3a.option.AllowDeleteBucketNotEmpty { + entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 1) + if err != nil { + return fmt.Errorf("failed to list bucket %s: %v", bucket, err) + } + if len(entries) > 0 { + return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code) + } + } // delete collection deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{ @@ -149,26 +172,36 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque return nil }) + if err != nil { + s3ErrorCode := s3err.ErrInternalError + if err.Error() == s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code { + s3ErrorCode = s3err.ErrBucketNotEmpty + } + s3err.WriteErrorResponse(w, r, s3ErrorCode) + return + } + err = s3a.rm(s3a.option.BucketsPath, bucket, false, true) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - s3err.WriteEmptyResponse(w, http.StatusNoContent) + s3err.WriteEmptyResponse(w, r, http.StatusNoContent) } func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("HeadBucketHandler %s", bucket) - if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { - s3err.WriteErrorResponse(w, err, r) + if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) return } - writeSuccessResponseEmpty(w) + writeSuccessResponseEmpty(w, r) } func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode { @@ -184,7 +217,7 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC } func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { - isAdmin := r.Header.Get(xhttp.AmzIsAdmin) != "" + isAdmin := r.Header.Get(s3_constants.AmzIsAdmin) != "" if isAdmin { return true } @@ -192,11 +225,119 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { return true } - identityId := r.Header.Get(xhttp.AmzIdentityId) - if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok { + identityId := r.Header.Get(s3_constants.AmzIdentityId) + if id, ok := entry.Extended[s3_constants.AmzIdentityId]; ok { if identityId != string(id) { return false } } return true } + +// GetBucketAclHandler Get Bucket ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html +func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) { + // collect parameters + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("GetBucketAclHandler %s", bucket) + + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + + response := AccessControlPolicy{} + for _, ident := range s3a.iam.identities { + if len(ident.Credentials) == 0 { + continue + } + for _, action := range ident.Actions { + if !action.overBucket(bucket) || action.getPermission() == "" { + continue + } + id := ident.Credentials[0].AccessKey + if response.Owner.DisplayName == "" && action.isOwner(bucket) && len(ident.Credentials) > 0 { + response.Owner.DisplayName = ident.Name + response.Owner.ID = id + } + response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{ + Grantee: Grantee{ + ID: id, + DisplayName: ident.Name, + Type: "CanonicalUser", + XMLXSI: "CanonicalUser", + XMLNS: "http://www.w3.org/2001/XMLSchema-instance"}, + Permission: action.getPermission(), + }) + } + } + writeSuccessResponseXML(w, r, response) +} + +// GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html +func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) { + // collect parameters + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("GetBucketLifecycleConfigurationHandler %s", bucket) + + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil) + if err != nil { + glog.Errorf("GetBucketLifecycleConfigurationHandler: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + ttls := fc.GetCollectionTtls(bucket) + if len(ttls) == 0 { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration) + return + } + response := Lifecycle{} + for prefix, internalTtl := range ttls { + ttl, _ := needle.ReadTTL(internalTtl) + days := int(ttl.Minutes() / 60 / 24) + if days == 0 { + continue + } + response.Rules = append(response.Rules, Rule{ + Status: Enabled, Filter: Filter{ + Prefix: Prefix{string: prefix, set: true}, + set: true, + }, + Expiration: Expiration{Days: days, set: true}, + }) + } + writeSuccessResponseXML(w, r, response) +} + +// PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html +func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) { + + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) + +} + +// DeleteBucketMetricsConfiguration Delete Bucket Lifecycle +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html +func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + + s3err.WriteEmptyResponse(w, r, http.StatusNoContent) + +} + +// GetBucketLocationHandler Get bucket location +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html +func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { + writeSuccessResponseXML(w, r, LocationConstraint{}) +} + +// GetBucketRequestPaymentHandler Get bucket location +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html +func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) { + writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"}) +} diff --git a/weed/s3api/s3api_bucket_skip_handlers.go b/weed/s3api/s3api_bucket_skip_handlers.go new file mode 100644 index 000000000..f4ca1177d --- /dev/null +++ b/weed/s3api/s3api_bucket_skip_handlers.go @@ -0,0 +1,49 @@ +package s3api + +import ( + "net/http" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" +) + +// GetBucketCorsHandler Get bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketCors.html +func (s3a *S3ApiServer) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchCORSConfiguration) +} + +// PutBucketCorsHandler Put bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html +func (s3a *S3ApiServer) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} + +// DeleteBucketCorsHandler Delete bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketCors.html +func (s3a *S3ApiServer) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, http.StatusNoContent) +} + +// GetBucketPolicyHandler Get bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicy.html +func (s3a *S3ApiServer) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucketPolicy) +} + +// PutBucketPolicyHandler Put bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketPolicy.html +func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} + +// DeleteBucketPolicyHandler Delete bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketPolicy.html +func (s3a *S3ApiServer) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, http.StatusNoContent) +} + +// PutBucketAclHandler Put bucket ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html +func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} diff --git a/weed/s3api/s3api_circuit_breaker.go b/weed/s3api/s3api_circuit_breaker.go new file mode 100644 index 000000000..111b404c7 --- /dev/null +++ b/weed/s3api/s3api_circuit_breaker.go @@ -0,0 +1,183 @@ +package s3api + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/gorilla/mux" + "net/http" + "sync" + "sync/atomic" +) + +type CircuitBreaker struct { + sync.RWMutex + Enabled bool + counters map[string]*int64 + limitations map[string]int64 +} + +func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker { + cb := &CircuitBreaker{ + counters: make(map[string]*int64), + limitations: make(map[string]int64), + } + + err := pb.WithFilerClient(false, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile) + if err != nil { + return fmt.Errorf("read S3 circuit breaker config: %v", err) + } + return cb.LoadS3ApiConfigurationFromBytes(content) + }) + + if err != nil { + glog.Warningf("fail to load config: %v", err) + } + + return cb +} + +func (cb *CircuitBreaker) LoadS3ApiConfigurationFromBytes(content []byte) error { + cbCfg := &s3_pb.S3CircuitBreakerConfig{} + if err := filer.ParseS3ConfigurationFromBytes(content, cbCfg); err != nil { + glog.Warningf("unmarshal error: %v", err) + return fmt.Errorf("unmarshal error: %v", err) + } + if err := cb.loadCircuitBreakerConfig(cbCfg); err != nil { + return err + } + return nil +} + +func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerConfig) error { + + //global + globalEnabled := false + globalOptions := cfg.Global + limitations := make(map[string]int64) + if globalOptions != nil && globalOptions.Enabled && len(globalOptions.Actions) > 0 { + globalEnabled = globalOptions.Enabled + for action, limit := range globalOptions.Actions { + limitations[action] = limit + } + } + cb.Enabled = globalEnabled + + //buckets + for bucket, cbOptions := range cfg.Buckets { + if cbOptions.Enabled { + for action, limit := range cbOptions.Actions { + limitations[s3_constants.Concat(bucket, action)] = limit + } + } + } + + cb.limitations = limitations + return nil +} + +func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), action string) (http.HandlerFunc, Action) { + return func(w http.ResponseWriter, r *http.Request) { + if !cb.Enabled { + f(w, r) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + rollback, errCode := cb.limit(r, bucket, action) + defer func() { + for _, rf := range rollback { + rf() + } + }() + + if errCode == s3err.ErrNone { + f(w, r) + return + } + s3err.WriteErrorResponse(w, r, errCode) + }, Action(action) +} + +func (cb *CircuitBreaker) limit(r *http.Request, bucket string, action string) (rollback []func(), errCode s3err.ErrorCode) { + + //bucket simultaneous request count + bucketCountRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(bucket, action, s3_constants.LimitTypeCount), 1, s3err.ErrTooManyRequest) + if bucketCountRollBack != nil { + rollback = append(rollback, bucketCountRollBack) + } + if errCode != s3err.ErrNone { + return + } + + //bucket simultaneous request content bytes + bucketContentLengthRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(bucket, action, s3_constants.LimitTypeBytes), r.ContentLength, s3err.ErrRequestBytesExceed) + if bucketContentLengthRollBack != nil { + rollback = append(rollback, bucketContentLengthRollBack) + } + if errCode != s3err.ErrNone { + return + } + + //global simultaneous request count + globalCountRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(action, s3_constants.LimitTypeCount), 1, s3err.ErrTooManyRequest) + if globalCountRollBack != nil { + rollback = append(rollback, globalCountRollBack) + } + if errCode != s3err.ErrNone { + return + } + + //global simultaneous request content bytes + globalContentLengthRollBack, errCode := cb.loadCounterAndCompare(s3_constants.Concat(action, s3_constants.LimitTypeBytes), r.ContentLength, s3err.ErrRequestBytesExceed) + if globalContentLengthRollBack != nil { + rollback = append(rollback, globalContentLengthRollBack) + } + if errCode != s3err.ErrNone { + return + } + return +} + +func (cb *CircuitBreaker) loadCounterAndCompare(key string, inc int64, errCode s3err.ErrorCode) (f func(), e s3err.ErrorCode) { + e = s3err.ErrNone + if max, ok := cb.limitations[key]; ok { + cb.RLock() + counter, exists := cb.counters[key] + cb.RUnlock() + + if !exists { + cb.Lock() + counter, exists = cb.counters[key] + if !exists { + var newCounter int64 + counter = &newCounter + cb.counters[key] = counter + } + cb.Unlock() + } + current := atomic.LoadInt64(counter) + if current+inc > max { + e = errCode + return + } else { + current := atomic.AddInt64(counter, inc) + f = func() { + atomic.AddInt64(counter, -inc) + } + if current > max { + e = errCode + return + } + } + } + return +} diff --git a/weed/s3api/s3api_circuit_breaker_test.go b/weed/s3api/s3api_circuit_breaker_test.go new file mode 100644 index 000000000..5848cf164 --- /dev/null +++ b/weed/s3api/s3api_circuit_breaker_test.go @@ -0,0 +1,107 @@ +package s3api + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "net/http" + "sync" + "sync/atomic" + "testing" +) + +type TestLimitCase struct { + actionName string + + limitType string + bucketLimitValue int64 + globalLimitValue int64 + + routineCount int + successCount int64 +} + +var ( + bucket = "/test" + action = s3_constants.ACTION_WRITE + fileSize int64 = 200 + + TestLimitCases = []*TestLimitCase{ + + //bucket-LimitTypeCount + {action, s3_constants.LimitTypeCount, 5, 6, 60, 5}, + {action, s3_constants.LimitTypeCount, 0, 6, 6, 0}, + + //global-LimitTypeCount + {action, s3_constants.LimitTypeCount, 6, 5, 6, 5}, + {action, s3_constants.LimitTypeCount, 6, 0, 6, 0}, + + //bucket-LimitTypeBytes + {action, s3_constants.LimitTypeBytes, 1000, 1020, 6, 5}, + {action, s3_constants.LimitTypeBytes, 0, 1020, 6, 0}, + + //global-LimitTypeBytes + {action, s3_constants.LimitTypeBytes, 1020, 1000, 6, 5}, + {action, s3_constants.LimitTypeBytes, 1020, 0, 6, 0}, + } +) + +func TestLimit(t *testing.T) { + for _, tc := range TestLimitCases { + circuitBreakerConfig := &s3_pb.S3CircuitBreakerConfig{ + Global: &s3_pb.S3CircuitBreakerOptions{ + Enabled: true, + Actions: map[string]int64{ + s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, + s3_constants.Concat(tc.actionName, tc.limitType): tc.globalLimitValue, + }, + }, + Buckets: map[string]*s3_pb.S3CircuitBreakerOptions{ + bucket: { + Enabled: true, + Actions: map[string]int64{ + s3_constants.Concat(tc.actionName, tc.limitType): tc.bucketLimitValue, + }, + }, + }, + } + circuitBreaker := &CircuitBreaker{ + counters: make(map[string]*int64), + limitations: make(map[string]int64), + } + err := circuitBreaker.loadCircuitBreakerConfig(circuitBreakerConfig) + if err != nil { + t.Fatal(err) + } + + successCount := doLimit(circuitBreaker, tc.routineCount, &http.Request{ContentLength: fileSize}, tc.actionName) + if successCount != tc.successCount { + t.Errorf("successCount not equal, expect=%d, actual=%d, case: %v", tc.successCount, successCount, tc) + } + } +} + +func doLimit(circuitBreaker *CircuitBreaker, routineCount int, r *http.Request, action string) int64 { + var successCounter int64 + resultCh := make(chan []func(), routineCount) + var wg sync.WaitGroup + for i := 0; i < routineCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + rollbackFn, errCode := circuitBreaker.limit(r, bucket, action) + if errCode == s3err.ErrNone { + atomic.AddInt64(&successCounter, 1) + } + resultCh <- rollbackFn + }() + } + wg.Wait() + close(resultCh) + for fns := range resultCh { + for _, fn := range fns { + fn() + } + } + return successCounter +} diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index b4d2c22e7..4ace4bb21 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -13,24 +13,26 @@ import ( var _ = filer_pb.FilerClient(&S3ApiServer{}) -func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (s3a *S3ApiServer) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) - }, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption) + }, s3a.option.Filer.ToGrpcAddress(), s3a.option.GrpcDialOption) } + func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { return location.Url } -func writeSuccessResponseXML(w http.ResponseWriter, response interface{}) { - s3err.WriteXMLResponse(w, http.StatusOK, response) +func writeSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) { + s3err.WriteXMLResponse(w, r, http.StatusOK, response) + s3err.PostLog(r, http.StatusOK, s3err.ErrNone) } -func writeSuccessResponseEmpty(w http.ResponseWriter) { - s3err.WriteEmptyResponse(w, http.StatusOK) +func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) { + s3err.WriteEmptyResponse(w, r, http.StatusOK) } func validateContentMd5(h http.Header) ([]byte, error) { diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 60dd54152..9157748f6 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -3,8 +3,9 @@ package s3api import ( "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - weed_server "github.com/chrislusf/seaweedfs/weed/server" + "modernc.org/strutil" "net/http" "net/url" "strconv" @@ -14,9 +15,14 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +const ( + DirectiveCopy = "COPY" + DirectiveReplace = "REPLACE" +) + func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { - dstBucket, dstObject := getBucketAndObject(r) + dstBucket, dstObject := s3_constants.GetBucketAndObject(r) // Copy source path. cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source")) @@ -27,19 +33,25 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request srcBucket, srcObject := pathToBucketAndObject(cpSrcPath) - if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) { + glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject) + + replaceMeta, replaceTagging := replaceDirective(r.Header) + + if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && (replaceMeta || replaceTagging) { fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)) dir, name := fullPath.DirAndName() entry, err := s3a.getEntry(dir, name) - if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + if err != nil || entry.IsDirectory { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) + return } - entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r)) + entry.Extended = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging) err = s3a.touch(dir, name, entry) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) + return } - writeSuccessResponseXML(w, CopyObjectResult{ + writeSuccessResponseXML(w, r, CopyObjectResult{ ETag: fmt.Sprintf("%x", entry.Attributes.Md5), LastModified: time.Now().UTC(), }) @@ -48,32 +60,44 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request // If source object is empty or bucket is empty, reply back invalid copy source. if srcObject == "" || srcBucket == "" { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) + return + } + srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject)) + dir, name := srcPath.DirAndName() + if entry, err := s3a.getEntry(dir, name); err != nil || entry.IsDirectory { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } if srcBucket == dstBucket && srcObject == dstObject { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopyDest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest) return } dstUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s", - s3a.option.Filer, s3a.option.BucketsPath, dstBucket, dstObject, dstBucket) + s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, dstBucket, urlPathEscape(dstObject), dstBucket) srcUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject) + s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject)) - _, _, resp, err := util.DownloadFile(srcUrl, "") + _, _, resp, err := util.DownloadFile(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false)) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } defer util.CloseResponse(resp) + tagErr := processMetadata(r.Header, resp.Header, replaceMeta, replaceTagging, s3a.getTags, dir, name) + if tagErr != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) + return + } glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body) + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) + etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } @@ -84,7 +108,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request LastModified: time.Now().UTC(), } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } @@ -105,7 +129,7 @@ type CopyPartResult struct { func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) { // https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html // https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html - dstBucket, _ := getBucketAndObject(r) + dstBucket, dstObject := s3_constants.GetBucketAndObject(r) // Copy source path. cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source")) @@ -117,7 +141,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req srcBucket, srcObject := pathToBucketAndObject(cpSrcPath) // If source object is empty or bucket is empty, reply back invalid copy source. if srcObject == "" || srcBucket == "" { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } @@ -126,35 +150,38 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req partID, err := strconv.Atoi(partIDString) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart) return } + glog.V(3).Infof("CopyObjectPartHandler %s %s => %s part %d", srcBucket, srcObject, dstBucket, partID) + // check partID with maximum part ID for multipart objects if partID > globalMaxPartID { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts) return } rangeHeader := r.Header.Get("x-amz-copy-source-range") dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s", - s3a.option.Filer, s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket) + s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket) srcUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject) + s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject)) - dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader) + dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false), rangeHeader) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } defer dataReader.Close() glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - etag, errCode := s3a.putToFiler(r, dstUrl, dataReader) + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) + etag, errCode := s3a.putToFiler(r, dstUrl, dataReader, destination) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } @@ -165,10 +192,111 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req LastModified: time.Now().UTC(), } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } -func isReplace(r *http.Request) bool { - return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE" +func replaceDirective(reqHeader http.Header) (replaceMeta, replaceTagging bool) { + return reqHeader.Get(s3_constants.AmzUserMetaDirective) == DirectiveReplace, reqHeader.Get(s3_constants.AmzObjectTaggingDirective) == DirectiveReplace +} + +func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTagging bool, getTags func(parentDirectoryPath string, entryName string) (tags map[string]string, err error), dir, name string) (err error) { + if sc := reqHeader.Get(s3_constants.AmzStorageClass); len(sc) == 0 { + if sc := existing[s3_constants.AmzStorageClass]; len(sc) > 0 { + reqHeader[s3_constants.AmzStorageClass] = sc + } + } + + if !replaceMeta { + for header, _ := range reqHeader { + if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) { + delete(reqHeader, header) + } + } + for k, v := range existing { + if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) { + reqHeader[k] = v + } + } + } + + if !replaceTagging { + for header, _ := range reqHeader { + if strings.HasPrefix(header, s3_constants.AmzObjectTagging) { + delete(reqHeader, header) + } + } + + found := false + for k, _ := range existing { + if strings.HasPrefix(k, s3_constants.AmzObjectTaggingPrefix) { + found = true + break + } + } + + if found { + tags, err := getTags(dir, name) + if err != nil { + return err + } + + var tagArr []string + for k, v := range tags { + tagArr = append(tagArr, fmt.Sprintf("%s=%s", k, v)) + } + tagStr := strutil.JoinFields(tagArr, "&") + reqHeader.Set(s3_constants.AmzObjectTagging, tagStr) + } + } + return +} + +func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte) { + metadata = make(map[string][]byte) + + if sc := existing[s3_constants.AmzStorageClass]; len(sc) > 0 { + metadata[s3_constants.AmzStorageClass] = sc + } + if sc := reqHeader.Get(s3_constants.AmzStorageClass); len(sc) > 0 { + metadata[s3_constants.AmzStorageClass] = []byte(sc) + } + + if replaceMeta { + for header, values := range reqHeader { + if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) { + for _, value := range values { + metadata[header] = []byte(value) + } + } + } + } else { + for k, v := range existing { + if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) { + metadata[k] = v + } + } + } + + if replaceTagging { + if tags := reqHeader.Get(s3_constants.AmzObjectTagging); tags != "" { + for _, v := range strings.Split(tags, "&") { + tag := strings.Split(v, "=") + if len(tag) == 2 { + metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) + } else if len(tag) == 1 { + metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = nil + } + } + } + } else { + for k, v := range existing { + if strings.HasPrefix(k, s3_constants.AmzObjectTagging) { + metadata[k] = v + } + } + delete(metadata, s3_constants.AmzTagCount) + } + + return } diff --git a/weed/s3api/s3api_object_copy_handlers_test.go b/weed/s3api/s3api_object_copy_handlers_test.go new file mode 100644 index 000000000..610b29a6b --- /dev/null +++ b/weed/s3api/s3api_object_copy_handlers_test.go @@ -0,0 +1,426 @@ +package s3api + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "net/http" + "reflect" + "sort" + "strings" + "testing" +) + +type H map[string]string + +func (h H) String() string { + pairs := make([]string, 0, len(h)) + for k, v := range h { + pairs = append(pairs, fmt.Sprintf("%s : %s", k, v)) + } + sort.Strings(pairs) + join := strings.Join(pairs, "\n") + return "\n" + join + "\n" +} + +var processMetadataTestCases = []struct { + caseId int + request H + existing H + getTags H + want H +}{ + { + 201, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging": "A=B&a=b&type=existing", + }, + }, + { + 202, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=existing", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + }, + }, + + { + 203, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + }, + + { + 204, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + }, + + { + 205, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{}, + H{}, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + }, + + { + 206, + H{ + "User-Agent": "firefox", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + }, + + { + 207, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-Type": "existing", + }, + H{ + "A": "B", + "a": "b", + "type": "existing", + }, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + }, +} +var processMetadataBytesTestCases = []struct { + caseId int + request H + existing H + want H +}{ + { + 101, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + }, + + { + 102, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + H{ + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + }, + + { + 103, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "request", + }, + }, + + { + 104, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + H{ + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "request", + }, + }, + + { + 105, + H{ + "User-Agent": "firefox", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{ + "X-Amz-Meta-My-Meta": "existing", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "existing", + }, + H{}, + }, + + { + 107, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{}, + H{ + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging-A": "B", + "X-Amz-Tagging-a": "b", + "X-Amz-Tagging-type": "request", + }, + }, +} + +func TestProcessMetadata(t *testing.T) { + for _, tc := range processMetadataTestCases { + reqHeader := transferHToHeader(tc.request) + existing := transferHToHeader(tc.existing) + replaceMeta, replaceTagging := replaceDirective(reqHeader) + + err := processMetadata(reqHeader, existing, replaceMeta, replaceTagging, func(_ string, _ string) (tags map[string]string, err error) { + return tc.getTags, nil + }, "", "") + if err != nil { + t.Error(err) + } + + result := transferHeaderToH(reqHeader) + fmtTagging(result, tc.want) + + if !reflect.DeepEqual(result, tc.want) { + t.Error(fmt.Errorf("\n### CaseID: %d ###"+ + "\nRequest:%v"+ + "\nExisting:%v"+ + "\nGetTags:%v"+ + "\nWant:%v"+ + "\nActual:%v", + tc.caseId, tc.request, tc.existing, tc.getTags, tc.want, result)) + } + } +} + +func TestProcessMetadataBytes(t *testing.T) { + for _, tc := range processMetadataBytesTestCases { + reqHeader := transferHToHeader(tc.request) + existing := transferHToBytesArr(tc.existing) + replaceMeta, replaceTagging := replaceDirective(reqHeader) + extends := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging) + + result := transferBytesArrToH(extends) + fmtTagging(result, tc.want) + + if !reflect.DeepEqual(result, tc.want) { + t.Error(fmt.Errorf("\n### CaseID: %d ###"+ + "\nRequest:%v"+ + "\nExisting:%v"+ + "\nWant:%v"+ + "\nActual:%v", + tc.caseId, tc.request, tc.existing, tc.want, result)) + } + } +} + +func fmtTagging(maps ...map[string]string) { + for _, m := range maps { + if tagging := m[s3_constants.AmzObjectTagging]; len(tagging) > 0 { + split := strings.Split(tagging, "&") + sort.Strings(split) + m[s3_constants.AmzObjectTagging] = strings.Join(split, "&") + } + } +} + +func transferHToHeader(data map[string]string) http.Header { + header := http.Header{} + for k, v := range data { + header.Add(k, v) + } + return header +} + +func transferHToBytesArr(data map[string]string) map[string][]byte { + m := make(map[string][]byte, len(data)) + for k, v := range data { + m[k] = []byte(v) + } + return m +} + +func transferBytesArrToH(data map[string][]byte) H { + m := make(map[string]string, len(data)) + for k, v := range data { + m[k] = string(v) + } + return m +} + +func transferHeaderToH(data map[string][]string) H { + m := make(map[string]string, len(data)) + for k, v := range data { + m[k] = v[len(v)-1] + } + return m +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 845c9a577..4ad3454ba 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -1,23 +1,25 @@ package s3api import ( + "bytes" "crypto/md5" "encoding/json" "encoding/xml" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/pquerna/cachecontrol/cacheobject" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util/mem" + "golang.org/x/exp/slices" "io" - "io/ioutil" "net/http" "net/url" - "sort" "strings" "time" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/pquerna/cachecontrol/cacheobject" - "github.com/gorilla/mux" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -25,39 +27,43 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -var ( - client *http.Client +const ( + deleteMultipleObjectsLimmit = 1000 ) -func init() { - client = &http.Client{Transport: &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - }} +func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser { + mimeBuffer := make([]byte, 512) + size, _ := dataReader.Read(mimeBuffer) + if size > 0 { + r.Header.Set("Content-Type", http.DetectContentType(mimeBuffer[:size])) + return io.NopCloser(io.MultiReader(bytes.NewReader(mimeBuffer[:size]), dataReader)) + } + return io.NopCloser(dataReader) } func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) { // http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutObjectHandler %s %s", bucket, object) _, err := validateContentMd5(r.Header) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest) return } if r.Header.Get("Cache-Control") != "" { if _, err = cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest) return } } if r.Header.Get("Expires") != "" { if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedExpires) return } } @@ -75,36 +81,45 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, s3ErrCode, r) + s3err.WriteErrorResponse(w, r, s3ErrCode) return } } else { if authTypeStreamingSigned == rAuthType { - s3err.WriteErrorResponse(w, s3err.ErrAuthNotSetup, r) + s3err.WriteErrorResponse(w, r, s3err.ErrAuthNotSetup) return } } defer dataReader.Close() + objectContentType := r.Header.Get("Content-Type") if strings.HasSuffix(object, "/") { - if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) { + if objectContentType == "" { + objectContentType = "httpd/unix-directory" + } + entry.Attributes.Mime = objectContentType + }); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } } else { - uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object)) + uploadUrl := s3a.toFilerUrl(bucket, object) + if objectContentType == "" { + dataReader = mimeDetect(r, dataReader) + } - etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, "") if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } setEtag(w, etag) } - writeSuccessResponseEmpty(w) + writeSuccessResponseEmpty(w, r) } func urlPathEscape(object string) string { @@ -115,45 +130,51 @@ func urlPathEscape(object string) string { return strings.Join(escapedParts, "/") } +func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { + destUrl := fmt.Sprintf("http://%s%s/%s%s", + s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) + return destUrl +} + func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("GetObjectHandler %s %s", bucket, object) if strings.HasSuffix(r.URL.Path, "/") { - s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) return } - destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object)) - - s3a.proxyToFiler(w, r, destUrl, passThroughResponse) + destUrl := s3a.toFilerUrl(bucket, object) + s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) - - destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object)) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object) - s3a.proxyToFiler(w, r, destUrl, passThroughResponse) + destUrl := s3a.toFilerUrl(bucket, object) + s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("DeleteObjectHandler %s %s", bucket, object) - destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true", - s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object)) + destUrl := s3a.toFilerUrl(bucket, object) - s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) { + s3a.proxyToFiler(w, r, destUrl, true, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) { + statusCode = http.StatusNoContent for k, v := range proxyResponse.Header { w.Header()[k] = v } - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(statusCode) + return statusCode }) } @@ -191,30 +212,39 @@ type DeleteObjectsResponse struct { // DeleteMultipleObjectsHandler - Delete multiple objects func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("DeleteMultipleObjectsHandler %s", bucket) - deleteXMLBytes, err := ioutil.ReadAll(r.Body) + deleteXMLBytes, err := io.ReadAll(r.Body) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } deleteObjects := &DeleteObjectsRequest{} if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + + if len(deleteObjects.Objects) > deleteMultipleObjectsLimmit { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxDeleteObjects) return } var deletedObjects []ObjectIdentifier var deleteErrors []DeleteError + var auditLog *s3err.AccessLog directoriesWithDeletion := make(map[string]int) - s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if s3err.Logger != nil { + auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone) + } + s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // delete file entries for _, object := range deleteObjects.Objects { - lastSeparator := strings.LastIndex(object.ObjectName, "/") parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.ObjectName, true, false if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) { @@ -237,6 +267,10 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h Key: object.ObjectName, }) } + if auditLog != nil { + auditLog.Key = entryName + s3err.PostAccessLog(*auditLog) + } } // purge empty folders, only checking folders with deletions @@ -253,7 +287,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h } deleteResp.Errors = deleteErrors - writeSuccessResponseXML(w, deleteResp) + writeSuccessResponseXML(w, r, deleteResp) } @@ -262,8 +296,8 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli for dir, _ := range directoriesWithDeletion { allDirs = append(allDirs, dir) } - sort.Slice(allDirs, func(i, j int) bool { - return len(allDirs[i]) > len(allDirs[j]) + slices.SortFunc(allDirs, func(a, b string) bool { + return len(a) > len(b) }) newDirectoriesWithDeletion = make(map[string]int) for _, dir := range allDirs { @@ -280,87 +314,81 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli return } -var passThroughHeaders = []string{ - "response-cache-control", - "response-content-disposition", - "response-content-encoding", - "response-content-language", - "response-content-type", - "response-expires", -} - -func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) { +func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, isWrite bool, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) { - glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl) + glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl) proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body) if err != nil { glog.Errorf("NewRequest %s: %v", destUrl, err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - proxyReq.Header.Set("Host", s3a.option.Filer) proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) - - for header, values := range r.Header { - // handle s3 related headers - passed := false - for _, h := range passThroughHeaders { - if strings.ToLower(header) == h && len(values) > 0 { - proxyReq.Header.Add(header[len("response-"):], values[0]) - passed = true - break - } - } - if passed { - continue - } - // handle other headers - for _, value := range values { - proxyReq.Header.Add(header, value) + for k, v := range r.URL.Query() { + if _, ok := s3_constants.PassThroughHeaders[strings.ToLower(k)]; ok { + proxyReq.Header[k] = v } } + for header, values := range r.Header { + proxyReq.Header[header] = values + } - resp, postErr := client.Do(proxyReq) + // ensure that the Authorization header is overriding any previous + // Authorization header which might be already present in proxyReq + s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite) + resp, postErr := s3a.client.Do(proxyReq) if postErr != nil { glog.Errorf("post to filer: %v", postErr) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } defer util.CloseResponse(resp) if resp.StatusCode == http.StatusPreconditionFailed { - s3err.WriteErrorResponse(w, s3err.ErrPreconditionFailed, r) + s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed) + return + } + + if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRange) return } if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 { if r.Method != "DELETE" { - s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) return } } - responseFn(resp, w) - + responseStatusCode := responseFn(resp, w) + s3err.PostLog(r, responseStatusCode, s3err.ErrNone) } -func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) { +func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) { for k, v := range proxyResponse.Header { w.Header()[k] = v } if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 { w.WriteHeader(http.StatusPartialContent) + statusCode = http.StatusPartialContent } else { - w.WriteHeader(proxyResponse.StatusCode) + statusCode = proxyResponse.StatusCode + } + w.WriteHeader(statusCode) + buf := mem.Allocate(128 * 1024) + defer mem.Free(buf) + if n, err := io.CopyBuffer(w, proxyResponse.Body, buf); err != nil { + glog.V(1).Infof("passthrough response read %d bytes: %v", n, err) } - io.Copy(w, proxyResponse.Body) + return statusCode } -func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) { +func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, destination string) (etag string, code s3err.ErrorCode) { hash := md5.New() var body = io.TeeReader(dataReader, hash) @@ -372,16 +400,20 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader return "", s3err.ErrInternalError } - proxyReq.Header.Set("Host", s3a.option.Filer) proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + if destination != "" { + proxyReq.Header.Set(s3_constants.SeaweedStorageDestinationHeader, destination) + } for header, values := range r.Header { for _, value := range values { proxyReq.Header.Add(header, value) } } - - resp, postErr := client.Do(proxyReq) + // ensure that the Authorization header is overriding any previous + // Authorization header which might be already present in proxyReq + s3a.maybeAddFilerJwtAuthorization(proxyReq, true) + resp, postErr := s3a.client.Do(proxyReq) if postErr != nil { glog.Errorf("post to filer: %v", postErr) @@ -391,7 +423,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader etag = fmt.Sprintf("%x", hash.Sum(nil)) - resp_body, ra_err := ioutil.ReadAll(resp.Body) + resp_body, ra_err := io.ReadAll(resp.Body) if ra_err != nil { glog.Errorf("upload to filer response read %d: %v", resp.StatusCode, ra_err) return etag, s3err.ErrInternalError @@ -420,20 +452,33 @@ func setEtag(w http.ResponseWriter, etag string) { } } -func getBucketAndObject(r *http.Request) (bucket, object string) { - vars := mux.Vars(r) - bucket = vars["bucket"] - object = vars["object"] - if !strings.HasPrefix(object, "/") { - object = "/" + object +func filerErrorToS3Error(errString string) s3err.ErrorCode { + switch { + case strings.HasPrefix(errString, "existing ") && strings.HasSuffix(errString, "is a directory"): + return s3err.ErrExistingObjectIsDirectory + case strings.HasSuffix(errString, "is a file"): + return s3err.ErrExistingObjectIsFile + default: + return s3err.ErrInternalError } +} - return +func (s3a *S3ApiServer) maybeAddFilerJwtAuthorization(r *http.Request, isWrite bool) { + encodedJwt := s3a.maybeGetFilerJwtAuthorizationToken(isWrite) + + if encodedJwt == "" { + return + } + + r.Header.Set("Authorization", "BEARER "+string(encodedJwt)) } -func filerErrorToS3Error(errString string) s3err.ErrorCode { - if strings.HasPrefix(errString, "existing ") && strings.HasSuffix(errString, "is a directory") { - return s3err.ErrExistingObjectIsDirectory +func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string { + var encodedJwt security.EncodedJwt + if isWrite { + encodedJwt = security.GenJwtForFilerServer(s3a.filerGuard.SigningKey, s3a.filerGuard.ExpiresAfterSec) + } else { + encodedJwt = security.GenJwtForFilerServer(s3a.filerGuard.ReadSigningKey, s3a.filerGuard.ReadExpiresAfterSec) } - return s3err.ErrInternalError + return string(encodedJwt) } diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index e1125689f..5704fcf38 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -5,16 +5,17 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api/policy" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "github.com/dustin/go-humanize" - "github.com/gorilla/mux" "io" - "io/ioutil" "mime/multipart" "net/http" "net/url" "strings" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/policy" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/dustin/go-humanize" + "github.com/gorilla/mux" ) func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { @@ -24,25 +25,27 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R bucket := mux.Vars(r)["bucket"] + glog.V(3).Infof("PostPolicyBucketHandler %s", bucket) + reader, err := r.MultipartReader() if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest) return } form, err := reader.ReadForm(int64(5 * humanize.MiByte)) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest) return } defer form.RemoveAll() fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest) return } if fileBody == nil { - s3err.WriteErrorResponse(w, s3err.ErrPOSTFileRequired, r) + s3err.WriteErrorResponse(w, r, s3err.ErrPOSTFileRequired) return } defer fileBody.Close() @@ -60,7 +63,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R if successRedirect != "" { redirectURL, err = url.Parse(successRedirect) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest) return } } @@ -68,13 +71,13 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Verify policy signature. errCode := s3a.iam.doesPolicySignatureMatch(formValues) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest) return } @@ -83,7 +86,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes)) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r) + s3err.WriteErrorResponse(w, r, s3err.ErrPostPolicyConditionInvalidFormat) return } @@ -99,23 +102,23 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R lengthRange := postPolicyForm.Conditions.ContentLengthRange if lengthRange.Valid { if fileSize < lengthRange.Min { - s3err.WriteErrorResponse(w, s3err.ErrEntityTooSmall, r) + s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooSmall) return } if fileSize > lengthRange.Max { - s3err.WriteErrorResponse(w, s3err.ErrEntityTooLarge, r) + s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooLarge) return } } } - uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, bucket, urlPathEscape(object)) + uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) - etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody) + etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody, "") if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } @@ -123,7 +126,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R // Replace raw query params.. redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag) w.Header().Set("Location", redirectURL.String()) - s3err.WriteEmptyResponse(w, http.StatusSeeOther) + s3err.WriteEmptyResponse(w, r, http.StatusSeeOther) return } @@ -138,18 +141,19 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R ETag: `"` + etag + `"`, Location: w.Header().Get("Location"), } - s3err.WriteXMLResponse(w, http.StatusCreated, resp) + s3err.WriteXMLResponse(w, r, http.StatusCreated, resp) + s3err.PostLog(r, http.StatusCreated, s3err.ErrNone) case "200": - s3err.WriteEmptyResponse(w, http.StatusOK) + s3err.WriteEmptyResponse(w, r, http.StatusOK) default: - writeSuccessResponseEmpty(w) + writeSuccessResponseEmpty(w, r) } } // Extract form fields and file data from a HTTP POST Policy func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) { - /// HTML Form values + // / HTML Form values fileName = "" // Canonicalize the form values into http.Header. @@ -172,7 +176,7 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, b.WriteString(v) } fileSize = int64(b.Len()) - filePart = ioutil.NopCloser(b) + filePart = io.NopCloser(b) return filePart, fileName, fileSize, formValues, nil } diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 308d9a5f5..768f4d180 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -1,15 +1,20 @@ package s3api import ( + "crypto/sha1" + "encoding/xml" "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - weed_server "github.com/chrislusf/seaweedfs/weed/server" + "io" "net/http" "net/url" "strconv" "strings" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + weed_server "github.com/chrislusf/seaweedfs/weed/server" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" ) @@ -23,7 +28,7 @@ const ( // NewMultipartUploadHandler - New multipart upload. func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) createMultipartUploadInput := &s3.CreateMultipartUploadInput{ Bucket: aws.String(bucket), @@ -36,49 +41,71 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http createMultipartUploadInput.Metadata[k] = aws.String(string(v)) } + contentType := r.Header.Get("Content-Type") + if contentType != "" { + createMultipartUploadInput.ContentType = &contentType + } response, errCode := s3a.createMultipartUpload(createMultipartUploadInput) glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } // CompleteMultipartUploadHandler - Completes multipart upload. func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html + + bucket, object := s3_constants.GetBucketAndObject(r) + + parts := &CompleteMultipartUpload{} + if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) + err := s3a.checkUploadId(object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), UploadId: aws.String(uploadID), - }) + }, parts) glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } // AbortMultipartUploadHandler - Aborts multipart upload. func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) + err := s3a.checkUploadId(object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{ Bucket: aws.String(bucket), @@ -87,29 +114,31 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht }) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response))) - writeSuccessResponseXML(w, response) + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html + s3err.WriteXMLResponse(w, r, http.StatusNoContent, response) + s3err.PostLog(r, http.StatusNoContent, s3err.ErrNone) } // ListMultipartUploadsHandler - Lists multipart uploads. func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) if maxUploads < 0 { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads) return } if keyMarker != "" { // Marker not common with prefix is not implemented. if !strings.HasPrefix(keyMarker, prefix) { - s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) return } } @@ -124,29 +153,35 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht UploadIdMarker: aws.String(uploadIDMarker), }) - glog.V(2).Info("ListMultipartUploadsHandler", string(s3err.EncodeXMLResponse(response)), errCode) + glog.V(2).Infof("ListMultipartUploadsHandler %s errCode=%d", string(s3err.EncodeXMLResponse(response)), errCode) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } // TODO handle encodingType - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } // ListObjectPartsHandler - Lists object parts in a multipart upload. func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query()) if partNumberMarker < 0 { - s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker) return } if maxParts < 0 { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts) + return + } + + err := s3a.checkUploadId(object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return } @@ -158,36 +193,36 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re UploadId: aws.String(uploadID), }) - glog.V(2).Info("ListObjectPartsHandler", string(s3err.EncodeXMLResponse(response)), errCode) - if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } - writeSuccessResponseXML(w, response) + glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part)) + + writeSuccessResponseXML(w, r, response) } // PutObjectPartHandler - Put an object part in a multipart upload. func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { - bucket, _ := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) uploadID := r.URL.Query().Get("uploadId") - exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) - if !exists { - s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r) + err := s3a.checkUploadId(object, uploadID) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return } partIDString := r.URL.Query().Get("partNumber") partID, err := strconv.Atoi(partIDString) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart) return } if partID > globalMaxPartID { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts) return } @@ -204,25 +239,31 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, s3ErrCode, r) + s3err.WriteErrorResponse(w, r, s3ErrCode) return } } defer dataReader.Close() + glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) + uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s", - s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID, bucket) + s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket) - etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) + if partID == 1 && r.Header.Get("Content-Type") == "" { + dataReader = mimeDetect(r, dataReader) + } + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, destination) if errCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, errCode, r) + s3err.WriteErrorResponse(w, r, errCode) return } setEtag(w, etag) - writeSuccessResponseEmpty(w) + writeSuccessResponseEmpty(w, r) } @@ -230,6 +271,27 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket) } +// Generate uploadID hash string from object +func (s3a *S3ApiServer) generateUploadID(object string) string { + if strings.HasPrefix(object, "/") { + object = object[1:] + } + h := sha1.New() + h.Write([]byte(object)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +//Check object name and uploadID when processing multipart uploading +func (s3a *S3ApiServer) checkUploadId(object string, id string) error { + + hash := s3a.generateUploadID(object) + if hash != id { + glog.Errorf("object %s and uploadID %s are not matched", object, id) + return fmt.Errorf("object %s and uploadID %s are not matched", object, id) + } + return nil +} + // Parse bucket url queries for ?uploads func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) { prefix = values.Get("prefix") @@ -258,8 +320,24 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m return } -type byCompletedPartNumber []*s3.CompletedPart +func xmlDecoder(body io.Reader, v interface{}, size int64) error { + var lbody io.Reader + if size > 0 { + lbody = io.LimitReader(body, size) + } else { + lbody = body + } + d := xml.NewDecoder(lbody) + d.CharsetReader = func(label string, input io.Reader) (io.Reader, error) { + return input, nil + } + return d.Decode(v) +} -func (a byCompletedPartNumber) Len() int { return len(a) } -func (a byCompletedPartNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byCompletedPartNumber) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber } +type CompleteMultipartUpload struct { + Parts []CompletedPart `xml:"Part"` +} +type CompletedPart struct { + ETag string + PartNumber int +} diff --git a/weed/s3api/s3api_object_skip_handlers.go b/weed/s3api/s3api_object_skip_handlers.go new file mode 100644 index 000000000..160d02475 --- /dev/null +++ b/weed/s3api/s3api_object_skip_handlers.go @@ -0,0 +1,45 @@ +package s3api + +import ( + "net/http" +) + +// GetObjectAclHandler Put object ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html +func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusNoContent) + +} + +// PutObjectAclHandler Put object ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html +func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusNoContent) + +} + +// PutObjectRetentionHandler Put object Retention +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html +func (s3a *S3ApiServer) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusNoContent) + +} + +// PutObjectLegalHoldHandler Put object Legal Hold +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html +func (s3a *S3ApiServer) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusNoContent) + +} + +// PutObjectLockConfigurationHandler Put object Lock configuration +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLockConfiguration.html +func (s3a *S3ApiServer) PutObjectLockConfigurationHandler(w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusNoContent) + +} diff --git a/weed/s3api/s3api_object_tagging_handlers.go b/weed/s3api/s3api_object_tagging_handlers.go index fd3ec2ff7..9fde0309c 100644 --- a/weed/s3api/s3api_object_tagging_handlers.go +++ b/weed/s3api/s3api_object_tagging_handlers.go @@ -3,20 +3,22 @@ package s3api import ( "encoding/xml" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "io" + "net/http" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/util" - "io" - "io/ioutil" - "net/http" ) // GetObjectTaggingHandler - GET object tagging // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("GetObjectTaggingHandler %s %s", bucket, object) target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) dir, name := target.DirAndName() @@ -25,15 +27,15 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R if err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) } else { glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) } return } - writeSuccessResponseXML(w, FromTags(tags)) + writeSuccessResponseXML(w, r, FromTags(tags)) } @@ -41,38 +43,39 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutObjectTaggingHandler %s %s", bucket, object) target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) dir, name := target.DirAndName() tagging := &Tagging{} - input, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength)) + input, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength)) if err != nil { glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } if err = xml.Unmarshal(input, tagging); err != nil { glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) return } tags := tagging.ToTags() if len(tags) > 10 { glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags)) - s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) return } for k, v := range tags { if len(k) > 128 { glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k) - s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) return } if len(v) > 256 { glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v) - s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) return } } @@ -80,23 +83,24 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) } else { glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) } return } - w.WriteHeader(http.StatusNoContent) - + w.WriteHeader(http.StatusOK) + s3err.PostLog(r, http.StatusOK, s3err.ErrNone) } // DeleteObjectTaggingHandler Delete object tagging // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := getBucketAndObject(r) + bucket, object := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("DeleteObjectTaggingHandler %s %s", bucket, object) target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) dir, name := target.DirAndName() @@ -105,13 +109,14 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt if err != nil { if err == filer_pb.ErrNotFound { glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) } else { glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err) - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) } return } w.WriteHeader(http.StatusNoContent) + s3err.PostLog(r, http.StatusNoContent, s3err.ErrNone) } diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 51a58af6a..6b934bccd 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "io" "net/http" "net/url" @@ -15,7 +16,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" ) @@ -39,16 +39,17 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html // collect parameters - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("ListObjectsV2Handler %s", bucket) originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query()) if maxKeys < 0 { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } if delimiter != "" && delimiter != "/" { - s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) return } @@ -60,13 +61,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } if len(response.Contents) == 0 { if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists { - s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) return } } @@ -86,7 +87,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ StartAfter: startAfter, } - writeSuccessResponseXML(w, responseV2) + writeSuccessResponseXML(w, r, responseV2) } func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { @@ -94,34 +95,35 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html // collect parameters - bucket, _ := getBucketAndObject(r) + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("ListObjectsV1Handler %s", bucket) originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query()) if maxKeys < 0 { - s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } if delimiter != "" && delimiter != "/" { - s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) return } response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) if err != nil { - s3err.WriteErrorResponse(w, s3err.ErrInternalError, r) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } if len(response.Contents) == 0 { if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists { - s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) return } } - writeSuccessResponseXML(w, response) + writeSuccessResponseXML(w, r, response) } func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) { @@ -131,10 +133,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m reqDir = reqDir[1:] } bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) + bucketPrefixLen := len(bucketPrefix) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) if strings.HasSuffix(reqDir, "/") { - // remove trailing "/" - reqDir = reqDir[:len(reqDir)-1] + reqDir = strings.TrimSuffix(reqDir, "/") } var contents []ListEntry @@ -144,33 +146,36 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m var nextMarker string // check filer - err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) { + _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) { if entry.IsDirectory { if delimiter == "/" { commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:], }) } - } else { - storageClass := "STANDARD" - if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { - storageClass = string(v) + if !(entry.IsDirectoryKeyObject() && strings.HasSuffix(entry.Name, "/")) { + return } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) } + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s", dir, entry.Name)[bucketPrefixLen:], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: StorageClass(storageClass), + }) }) + glog.V(4).Infof("end doListFilerEntries isTruncated:%v nextMarker:%v reqDir: %v prefix: %v", isTruncated, nextMarker, reqDir, prefix) if doErr != nil { return doErr } @@ -179,6 +184,39 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m nextMarker = "" } + if len(contents) == 0 && len(commonPrefixes) == 0 && maxKeys > 0 { + if strings.HasSuffix(originalPrefix, "/") && prefix == "" { + reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/")) + reqDir = strings.TrimSuffix(reqDir, "/") + } + _, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) { + if entry.IsDirectoryKeyObject() && entry.Name == prefix { + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + fmt.Sprintf("%x", entry.Attributes.Md5) + "\"", + Size: int64(filer.FileSize(entry)), + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: StorageClass(storageClass), + }) + } + }) + if doErr != nil { + return doErr + } + } + + if len(nextMarker) > 0 { + nextMarker = nextMarker[bucketPrefixLen:] + } + response = ListBucketResult{ Name: bucket, Prefix: originalPrefix, @@ -197,7 +235,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m return } -func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { +func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, subEntries bool, bucketPrefixLen int, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { // invariants // prefix and marker should be under dir, marker may contain "/" // maxKeys should be updated for each recursion @@ -210,19 +248,30 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } if strings.Contains(marker, "/") { + if strings.HasSuffix(marker, "/") { + marker = strings.TrimSuffix(marker, "/") + } sepIndex := strings.Index(marker, "/") - subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:] - // println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn) - if subErr != nil { - err = subErr - return + if sepIndex != -1 { + subPrefix, subMarker := marker[0:sepIndex], marker[sepIndex+1:] + subDir := fmt.Sprintf("%s/%s", dir[0:bucketPrefixLen-1], subPrefix) + if strings.HasPrefix(subDir, dir) { + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, subDir, "", maxKeys, subMarker, delimiter, false, false, bucketPrefixLen, eachEntryFn) + if subErr != nil { + err = subErr + return + } + counter += subCounter + isTruncated = isTruncated || subIsTruncated + maxKeys -= subCounter + nextMarker = subNextMarker + // finished processing this sub directory + marker = subPrefix + } } - isTruncated = isTruncated || subIsTruncated - maxKeys -= subCounter - nextMarker = subDir + "/" + subNextMarker - // finished processing this sub directory - marker = subDir + } + if maxKeys <= 0 { + return } // now marker is also a direct child of dir @@ -231,7 +280,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d Prefix: prefix, Limit: uint32(maxKeys + 1), StartFromFileName: marker, - InclusiveStartFrom: false, + InclusiveStartFrom: inclusiveStartFrom, } ctx, cancel := context.WithCancel(context.Background()) @@ -257,39 +306,46 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } entry := resp.Entry - nextMarker = entry.Name + nextMarker = dir + "/" + entry.Name if entry.IsDirectory { // println("ListEntries", dir, "dir:", entry.Name) - if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys - if delimiter != "/" { + if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys + continue + } + if delimiter == "" { + eachEntryFn(dir, entry) + // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, true, bucketPrefixLen, eachEntryFn) + if subErr != nil { + err = fmt.Errorf("doListFilerEntries2: %v", subErr) + return + } + // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated) + if subCounter == 0 && entry.IsDirectoryKeyObject() { + entry.Name += "/" eachEntryFn(dir, entry) - // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn) - if subErr != nil { - err = fmt.Errorf("doListFilerEntries2: %v", subErr) - return - } - // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated) - counter += subCounter - nextMarker = entry.Name + "/" + subNextMarker - if subIsTruncated { - isTruncated = true - return - } - } else { - var isEmpty bool - if !s3a.option.AllowEmptyFolder { - if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { - glog.Errorf("check empty folder %s: %v", dir, err) - } - } - if !isEmpty { - eachEntryFn(dir, entry) - counter++ + counter++ + } + counter += subCounter + nextMarker = subNextMarker + if subIsTruncated { + isTruncated = true + return + } + } else if delimiter == "/" { + var isEmpty bool + if !s3a.option.AllowEmptyFolder && !entry.IsDirectoryKeyObject() { + if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { + glog.Errorf("check empty folder %s: %v", dir, err) } } + if !isEmpty { + nextMarker += "/" + eachEntryFn(dir, entry) + counter++ + } } - } else { + } else if !(delimiter == "/" && subEntries) { // println("ListEntries", dir, "file:", entry.Name) eachEntryFn(dir, entry) counter++ diff --git a/weed/s3api/s3api_policy.go b/weed/s3api/s3api_policy.go new file mode 100644 index 000000000..6e2c8cfa2 --- /dev/null +++ b/weed/s3api/s3api_policy.go @@ -0,0 +1,148 @@ +package s3api + +import ( + "encoding/xml" + "time" +) + +// Status represents lifecycle configuration status +type ruleStatus string + +// Supported status types +const ( + Enabled ruleStatus = "Enabled" + Disabled ruleStatus = "Disabled" +) + +// Lifecycle - Configuration for bucket lifecycle. +type Lifecycle struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + Rules []Rule `xml:"Rule"` +} + +// Rule - a rule for lifecycle configuration. +type Rule struct { + XMLName xml.Name `xml:"Rule"` + ID string `xml:"ID,omitempty"` + Status ruleStatus `xml:"Status"` + Filter Filter `xml:"Filter,omitempty"` + Prefix Prefix `xml:"Prefix,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + Transition Transition `xml:"Transition,omitempty"` +} + +// Filter - a filter for a lifecycle configuration Rule. +type Filter struct { + XMLName xml.Name `xml:"Filter"` + set bool + + Prefix Prefix + + And And + andSet bool + + Tag Tag + tagSet bool +} + +// Prefix holds the prefix xml tag in and +type Prefix struct { + string + set bool +} + +// MarshalXML encodes Prefix field into an XML form. +func (p Prefix) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !p.set { + return nil + } + return e.EncodeElement(p.string, startElement) +} + +// MarshalXML encodes Filter field into an XML form. +func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule. +type And struct { + XMLName xml.Name `xml:"And"` + Prefix Prefix `xml:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty"` +} + +// Expiration - expiration actions for a rule in lifecycle configuration. +type Expiration struct { + XMLName xml.Name `xml:"Expiration"` + Days int `xml:"Days,omitempty"` + Date ExpirationDate `xml:"Date,omitempty"` + DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker"` + + set bool +} + +// MarshalXML encodes expiration field into an XML form. +func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement) error { + if !e.set { + return nil + } + type expirationWrapper Expiration + return enc.EncodeElement(expirationWrapper(e), startElement) +} + +// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. +type ExpireDeleteMarker struct { + val bool + set bool +} + +// MarshalXML encodes delete marker boolean into an XML form. +func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !b.set { + return nil + } + return e.EncodeElement(b.val, startElement) +} + +// ExpirationDate is a embedded type containing time.Time to unmarshal +// Date in Expiration +type ExpirationDate struct { + time.Time +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDate.Time.IsZero() { + return nil + } + return e.EncodeElement(eDate.Format(time.RFC3339), startElement) +} + +// Transition - transition actions for a rule in lifecycle configuration. +type Transition struct { + XMLName xml.Name `xml:"Transition"` + Days int `xml:"Days,omitempty"` + Date time.Time `xml:"Date,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + + set bool +} + +// MarshalXML encodes transition field into an XML form. +func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !t.set { + return nil + } + type transitionWrapper Transition + return enc.EncodeElement(transitionWrapper(t), start) +} + +// TransitionDays is a type alias to unmarshal Days in Transition +type TransitionDays int diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 27259c4a7..cc5ca5231 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1,44 +1,81 @@ package s3api import ( + "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" + "net" "net/http" "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb" . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" "github.com/gorilla/mux" "google.golang.org/grpc" ) type S3ApiServerOption struct { - Filer string - Port int - FilerGrpcAddress string - Config string - DomainName string - BucketsPath string - GrpcDialOption grpc.DialOption - AllowEmptyFolder bool + Filer pb.ServerAddress + Port int + Config string + DomainName string + BucketsPath string + GrpcDialOption grpc.DialOption + AllowEmptyFolder bool + AllowDeleteBucketNotEmpty bool + LocalFilerSocket *string } type S3ApiServer struct { - option *S3ApiServerOption - iam *IdentityAccessManagement + s3_pb.UnimplementedSeaweedS3Server + option *S3ApiServerOption + iam *IdentityAccessManagement + cb *CircuitBreaker + randomClientId int32 + filerGuard *security.Guard + client *http.Client } func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) { + v := util.GetViper() + signingKey := v.GetString("jwt.filer_signing.key") + v.SetDefault("jwt.filer_signing.expires_after_seconds", 10) + expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds") + + readSigningKey := v.GetString("jwt.filer_signing.read.key") + v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) + readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + s3ApiServer = &S3ApiServer{ - option: option, - iam: NewIdentityAccessManagement(option), + option: option, + iam: NewIdentityAccessManagement(option), + randomClientId: util.RandomInt32(), + filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec), + cb: NewCircuitBreaker(option), + } + if option.LocalFilerSocket == nil || *option.LocalFilerSocket == "" { + s3ApiServer.client = &http.Client{Transport: &http.Transport{ + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 1024, + }} + } else { + s3ApiServer.client = &http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", *option.LocalFilerSocket) + }, + }, + } } s3ApiServer.registerRouter(router) - go s3ApiServer.subscribeMetaEvents("s3", filer.IamConfigDirecotry+"/"+filer.IamIdentityFile, time.Now().UnixNano()) - + go s3ApiServer.subscribeMetaEvents("s3", filer.DirectoryEtcRoot, time.Now().UnixNano()) return s3ApiServer, nil } @@ -63,73 +100,126 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { for _, bucket := range routers { - // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET")) - // HeadBucket - bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_ADMIN), "GET")) + // each case should follow the next rule: + // - requesting object with query must precede any other methods + // - requesting object must precede any methods with buckets + // - requesting bucket with query must precede raw methods with buckets + // - requesting bucket must be processed in the end + + // objects with query // CopyObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.CompleteMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.NewMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploads", "") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "") // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.AbortMultipartUploadHandler, ACTION_WRITE), "DELETE")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}") // ListObjectParts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectPartsHandler, ACTION_READ), "GET")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}") // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListMultipartUploadsHandler, ACTION_READ), "GET")).Queries("uploads", "") + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "") // GetObjectTagging - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectTaggingHandler, ACTION_READ), "GET")).Queries("tagging", "") + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "") // PutObjectTagging - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectTaggingHandler, ACTION_TAGGING), "PUT")).Queries("tagging", "") + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectTaggingHandler, ACTION_TAGGING)), "PUT")).Queries("tagging", "") // DeleteObjectTagging - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING), "DELETE")).Queries("tagging", "") + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "") + + // PutObjectACL + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") + // PutObjectRetention + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "") + // PutObjectLegalHold + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLegalHoldHandler, ACTION_WRITE)), "PUT")).Queries("legal-hold", "") + // PutObjectLockConfiguration + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "") + + // GetObjectACL + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ)), "GET")).Queries("acl", "") + + // objects with query + + // raw objects + + // HeadObject + bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET")) + + // GetObject, but directory listing is not supported + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET")) // CopyObject - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY")) + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY")) // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT")) - // PutBucket - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketHandler, ACTION_ADMIN), "PUT")) - + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT")) // DeleteObject - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE")) - // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE")) + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE")) + + // raw objects + + // buckets with query + + // DeleteMultipleObjects + bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "") + + // GetBucketACL + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ)), "GET")).Queries("acl", "") + // PutBucketACL + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") + + // GetBucketPolicy + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "") + // PutBucketPolicy + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketPolicyHandler, ACTION_WRITE)), "PUT")).Queries("policy", "") + // DeleteBucketPolicy + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)), "DELETE")).Queries("policy", "") + + // GetBucketCors + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketCorsHandler, ACTION_READ)), "GET")).Queries("cors", "") + // PutBucketCors + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketCorsHandler, ACTION_WRITE)), "PUT")).Queries("cors", "") + // DeleteBucketCors + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketCorsHandler, ACTION_WRITE)), "DELETE")).Queries("cors", "") + + // GetBucketLifecycleConfiguration + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "") + // PutBucketLifecycleConfiguration + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "") + // DeleteBucketLifecycleConfiguration + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "") + + // GetBucketLocation + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "") + + // GetBucketRequestPayment + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "") // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_LIST), "LIST")).Queries("list-type", "2") - // GetObject, but directory listing is not supported - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET")) - // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST")) + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") + + // buckets with query + + // raw buckets // PostPolicy - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.PostPolicyBucketHandler, ACTION_WRITE), "POST")) + bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PostPolicyBucketHandler, ACTION_WRITE)), "POST")) - // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "") - /* - - // not implemented - // GetBucketLocation - bucket.Methods("GET").HandlerFunc(s3a.GetBucketLocationHandler).Queries("location", "") - // GetBucketPolicy - bucket.Methods("GET").HandlerFunc(s3a.GetBucketPolicyHandler).Queries("policy", "") - // GetObjectACL - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectACLHandler).Queries("acl", "") - // GetBucketACL - bucket.Methods("GET").HandlerFunc(s3a.GetBucketACLHandler).Queries("acl", "") - // PutBucketPolicy - bucket.Methods("PUT").HandlerFunc(s3a.PutBucketPolicyHandler).Queries("policy", "") - // DeleteBucketPolicy - bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketPolicyHandler).Queries("policy", "") - */ + // HeadBucket + bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadBucketHandler, ACTION_READ)), "GET")) + + // PutBucket + bucket.Methods("PUT").HandlerFunc(track(s3a.PutBucketHandler, "PUT")) + // DeleteBucket + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_WRITE)), "DELETE")) + + // ListObjectsV1 (Legacy) + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) + + // raw buckets } diff --git a/weed/s3api/s3api_server_grpc.go b/weed/s3api/s3api_server_grpc.go new file mode 100644 index 000000000..e93d0056f --- /dev/null +++ b/weed/s3api/s3api_server_grpc.go @@ -0,0 +1,16 @@ +package s3api + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" +) + +func (s3a *S3ApiServer) Configure(ctx context.Context, request *s3_pb.S3ConfigureRequest) (*s3_pb.S3ConfigureResponse, error) { + + if err := s3a.iam.LoadS3ApiConfigurationFromBytes(request.S3ConfigurationFileContent); err != nil { + return nil, err + } + + return &s3_pb.S3ConfigureResponse{}, nil + +} diff --git a/weed/s3api/s3api_status_handlers.go b/weed/s3api/s3api_status_handlers.go index 914c27f40..fafb6ac2f 100644 --- a/weed/s3api/s3api_status_handlers.go +++ b/weed/s3api/s3api_status_handlers.go @@ -1,8 +1,11 @@ package s3api -import "net/http" +import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "net/http" +) func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) { // write out the response code and content type header - writeSuccessResponseEmpty(w) + s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "") } diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go index 9d62afc4e..dd6a32ff2 100644 --- a/weed/s3api/s3api_xsd_generated.go +++ b/weed/s3api/s3api_xsd_generated.go @@ -8,12 +8,12 @@ import ( ) type AccessControlList struct { - Grant []Grant `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Grant,omitempty"` + Grant []Grant `xml:"Grant,omitempty"` } type AccessControlPolicy struct { - Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList"` + Owner CanonicalUser `xml:"Owner"` + AccessControlList AccessControlList `xml:"AccessControlList"` } type AmazonCustomerByEmail struct { @@ -467,11 +467,17 @@ func (t *GetObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e } type Grant struct { - Grantee Grantee `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Grantee"` - Permission Permission `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Permission"` + Grantee Grantee `xml:"Grantee"` + Permission Permission `xml:"Permission"` } type Grantee struct { + XMLNS string `xml:"xmlns:xsi,attr"` + XMLXSI string `xml:"xsi:type,attr"` + Type string `xml:"Type"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI string `xml:"URI,omitempty"` } type Group struct { @@ -640,6 +646,10 @@ type ListVersionsResult struct { CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"` } +type LocationConstraint struct { + LocationConstraint string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint"` +} + type LoggingSettings struct { TargetBucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetBucket"` TargetPrefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetPrefix"` diff --git a/weed/s3api/s3err/audit_fluent.go b/weed/s3api/s3err/audit_fluent.go new file mode 100644 index 000000000..2deb56896 --- /dev/null +++ b/weed/s3api/s3err/audit_fluent.go @@ -0,0 +1,183 @@ +package s3err + +import ( + "encoding/json" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/fluent/fluent-logger-golang/fluent" + "net/http" + "os" + "time" +) + +type AccessLogExtend struct { + AccessLog + AccessLogHTTP +} + +type AccessLog struct { + Bucket string `msg:"bucket" json:"bucket"` // awsexamplebucket1 + Time int64 `msg:"time" json:"time"` // [06/Feb/2019:00:00:38 +0000] + RemoteIP string `msg:"remote_ip" json:"remote_ip,omitempty"` // 192.0.2.3 + Requester string `msg:"requester" json:"requester,omitempty"` // IAM user id + RequestID string `msg:"request_id" json:"request_id,omitempty"` // 3E57427F33A59F07 + Operation string `msg:"operation" json:"operation,omitempty"` // REST.HTTP_method.resource_type REST.PUT.OBJECT + Key string `msg:"key" json:"key,omitempty"` // /photos/2019/08/puppy.jpg + ErrorCode string `msg:"error_code" json:"error_code,omitempty"` + HostId string `msg:"host_id" json:"host_id,omitempty"` + HostHeader string `msg:"host_header" json:"host_header,omitempty"` // s3.us-west-2.amazonaws.com + UserAgent string `msg:"user_agent" json:"user_agent,omitempty"` + HTTPStatus int `msg:"status" json:"status,omitempty"` + SignatureVersion string `msg:"signature_version" json:"signature_version,omitempty"` +} + +type AccessLogHTTP struct { + RequestURI string `json:"request_uri,omitempty"` // "GET /awsexamplebucket1/photos/2019/08/puppy.jpg?x-foo=bar HTTP/1.1" + BytesSent string `json:"bytes_sent,omitempty"` + ObjectSize string `json:"object_size,omitempty"` + TotalTime int `json:"total_time,omitempty"` + TurnAroundTime int `json:"turn_around_time,omitempty"` + Referer string `json:"Referer,omitempty"` + VersionId string `json:"version_id,omitempty"` + CipherSuite string `json:"cipher_suite,omitempty"` + AuthenticationType string `json:"auth_type,omitempty"` + TLSVersion string `json:"TLS_version,omitempty"` +} + +const tag = "s3.access" + +var ( + Logger *fluent.Fluent + hostname = os.Getenv("HOSTNAME") + environment = os.Getenv("ENVIRONMENT") +) + +func InitAuditLog(config string) { + configContent, readErr := os.ReadFile(config) + if readErr != nil { + glog.Errorf("fail to read fluent config %s : %v", config, readErr) + return + } + fluentConfig := &fluent.Config{} + if err := json.Unmarshal(configContent, fluentConfig); err != nil { + glog.Errorf("fail to parse fluent config %s : %v", string(configContent), err) + return + } + if len(fluentConfig.TagPrefix) == 0 && len(environment) > 0 { + fluentConfig.TagPrefix = environment + } + fluentConfig.Async = true + fluentConfig.AsyncResultCallback = func(data []byte, err error) { + if err != nil { + glog.Warning("Error while posting log: ", err) + } + } + var err error + Logger, err = fluent.New(*fluentConfig) + if err != nil { + glog.Errorf("fail to load fluent config: %v", err) + } +} + +func getREST(httpMetod string, resourceType string) string { + return fmt.Sprintf("REST.%s.%s", httpMetod, resourceType) +} + +func getResourceType(object string, query_key string, metod string) (string, bool) { + if object == "/" { + switch query_key { + case "delete": + return "BATCH.DELETE.OBJECT", true + case "tagging": + return getREST(metod, "OBJECTTAGGING"), true + case "lifecycle": + return getREST(metod, "LIFECYCLECONFIGURATION"), true + case "acl": + return getREST(metod, "ACCESSCONTROLPOLICY"), true + case "policy": + return getREST(metod, "BUCKETPOLICY"), true + default: + return getREST(metod, "BUCKET"), false + } + } else { + switch query_key { + case "tagging": + return getREST(metod, "OBJECTTAGGING"), true + default: + return getREST(metod, "OBJECT"), false + } + } +} + +func getOperation(object string, r *http.Request) string { + queries := r.URL.Query() + var operation string + var queryFound bool + for key, _ := range queries { + operation, queryFound = getResourceType(object, key, r.Method) + if queryFound { + return operation + } + } + if len(queries) == 0 { + operation, _ = getResourceType(object, "", r.Method) + } + return operation +} + +func GetAccessHttpLog(r *http.Request, statusCode int, s3errCode ErrorCode) AccessLogHTTP { + return AccessLogHTTP{ + RequestURI: r.RequestURI, + Referer: r.Header.Get("Referer"), + } +} + +func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *AccessLog { + bucket, key := s3_constants.GetBucketAndObject(r) + var errorCode string + if s3errCode != ErrNone { + errorCode = GetAPIError(s3errCode).Code + } + remoteIP := r.Header.Get("X-Real-IP") + if len(remoteIP) == 0 { + remoteIP = r.RemoteAddr + } + hostHeader := r.Header.Get("X-Forwarded-Host") + if len(hostHeader) == 0 { + hostHeader = r.Host + } + return &AccessLog{ + HostHeader: hostHeader, + RequestID: r.Header.Get("X-Request-ID"), + RemoteIP: remoteIP, + Requester: r.Header.Get(s3_constants.AmzIdentityId), + SignatureVersion: r.Header.Get(s3_constants.AmzAuthType), + UserAgent: r.Header.Get("user-agent"), + HostId: hostname, + Bucket: bucket, + HTTPStatus: HTTPStatusCode, + Time: time.Now().Unix(), + Key: key, + Operation: getOperation(key, r), + ErrorCode: errorCode, + } +} + +func PostLog(r *http.Request, HTTPStatusCode int, errorCode ErrorCode) { + if Logger == nil { + return + } + if err := Logger.Post(tag, *GetAccessLog(r, HTTPStatusCode, errorCode)); err != nil { + glog.Warning("Error while posting log: ", err) + } +} + +func PostAccessLog(log AccessLog) { + if Logger == nil || len(log.Key) == 0 { + return + } + if err := Logger.Post(tag, log); err != nil { + glog.Warning("Error while posting log: ", err) + } +} diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go index c1065fffc..6753a1641 100644 --- a/weed/s3api/s3err/error_handler.go +++ b/weed/s3api/s3err/error_handler.go @@ -19,15 +19,16 @@ const ( MimeXML mimeType = "application/xml" ) -func WriteXMLResponse(w http.ResponseWriter, statusCode int, response interface{}) { - WriteResponse(w, statusCode, EncodeXMLResponse(response), MimeXML) +func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) { + WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML) } -func WriteEmptyResponse(w http.ResponseWriter, statusCode int) { - WriteResponse(w, statusCode, []byte{}, mimeNone) +func WriteEmptyResponse(w http.ResponseWriter, r *http.Request, statusCode int) { + WriteResponse(w, r, statusCode, []byte{}, mimeNone) + PostLog(r, statusCode, ErrNone) } -func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Request) { +func WriteErrorResponse(w http.ResponseWriter, r *http.Request, errorCode ErrorCode) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -38,7 +39,8 @@ func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Requ apiError := GetAPIError(errorCode) errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object) encodedErrorResponse := EncodeXMLResponse(errorResponse) - WriteResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML) + WriteResponse(w, r, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML) + PostLog(r, apiError.HTTPStatusCode, errorCode) } func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse { @@ -61,13 +63,17 @@ func EncodeXMLResponse(response interface{}) []byte { return bytesBuffer.Bytes() } -func setCommonHeaders(w http.ResponseWriter) { +func setCommonHeaders(w http.ResponseWriter, r *http.Request) { w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano())) w.Header().Set("Accept-Ranges", "bytes") + if r.Header.Get("Origin") != "" { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Credentials", "true") + } } -func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { - setCommonHeaders(w) +func WriteResponse(w http.ResponseWriter, r *http.Request, statusCode int, response []byte, mType mimeType) { + setCommonHeaders(w, r) if response != nil { w.Header().Set("Content-Length", strconv.Itoa(len(response))) } @@ -88,5 +94,5 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType // If none of the http routes match respond with MethodNotAllowed func NotFoundHandler(w http.ResponseWriter, r *http.Request) { glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI) - WriteErrorResponse(w, ErrMethodNotAllowed, r) + WriteErrorResponse(w, r, ErrMethodNotAllowed) } diff --git a/weed/s3api/s3err/s3-error.go b/weed/s3api/s3err/s3-error.go index 224378ec5..b87764742 100644 --- a/weed/s3api/s3err/s3-error.go +++ b/weed/s3api/s3err/s3-error.go @@ -58,4 +58,5 @@ var s3ErrorResponseMap = map[string]string{ "InvalidDuration": "Duration provided in the request is invalid.", "XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.", // Add new API errors here. + "NoSuchCORSConfiguration": "The CORS configuration does not exist", } diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index a46bd0f04..57f269a2e 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -51,6 +51,9 @@ const ( ErrBucketAlreadyExists ErrBucketAlreadyOwnedByYou ErrNoSuchBucket + ErrNoSuchBucketPolicy + ErrNoSuchCORSConfiguration + ErrNoSuchLifecycleConfiguration ErrNoSuchKey ErrNoSuchUpload ErrInvalidBucketName @@ -58,8 +61,10 @@ const ( ErrInvalidMaxKeys ErrInvalidMaxUploads ErrInvalidMaxParts + ErrInvalidMaxDeleteObjects ErrInvalidPartNumberMarker ErrInvalidPart + ErrInvalidRange ErrInternalError ErrInvalidCopyDest ErrInvalidCopySource @@ -98,6 +103,10 @@ const ( ErrPreconditionFailed ErrExistingObjectIsDirectory + ErrExistingObjectIsFile + + ErrTooManyRequest + ErrRequestBytesExceed ) // error code to APIError structure, these fields carry respective @@ -153,6 +162,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Argument max-parts must be an integer between 0 and 2147483647", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidMaxDeleteObjects: { + Code: "InvalidArgument", + Description: "Argument objects can contain a list of up to 1000 keys", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidPartNumberMarker: { Code: "InvalidArgument", Description: "Argument partNumberMarker must be an integer.", @@ -163,6 +177,21 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The specified bucket does not exist", HTTPStatusCode: http.StatusNotFound, }, + ErrNoSuchBucketPolicy: { + Code: "NoSuchBucketPolicy", + Description: "The bucket policy does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchCORSConfiguration: { + Code: "NoSuchCORSConfiguration", + Description: "The CORS configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchLifecycleConfiguration: { + Code: "NoSuchLifecycleConfiguration", + Description: "The lifecycle configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, ErrNoSuchKey: { Code: "NoSuchKey", Description: "The specified key does not exist.", @@ -196,7 +225,7 @@ var errorCodeResponse = map[ErrorCode]APIError{ HTTPStatusCode: http.StatusBadRequest, }, ErrInvalidTag: { - Code: "InvalidArgument", + Code: "InvalidTag", Description: "The Tag value you have provided is invalid", HTTPStatusCode: http.StatusBadRequest, }, @@ -345,6 +374,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Invalid Request", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidRange: { + Code: "InvalidRange", + Description: "The requested range is not satisfiable", + HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, + }, ErrAuthNotSetup: { Code: "InvalidRequest", Description: "Signed request requires setting up SeaweedFS S3 authentication", @@ -365,6 +399,21 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Existing Object is a directory.", HTTPStatusCode: http.StatusConflict, }, + ErrExistingObjectIsFile: { + Code: "ExistingObjectIsFile", + Description: "Existing Object is a file.", + HTTPStatusCode: http.StatusConflict, + }, + ErrTooManyRequest: { + Code: "ErrTooManyRequest", + Description: "Too many simultaneous request count", + HTTPStatusCode: http.StatusTooManyRequests, + }, + ErrRequestBytesExceed: { + Code: "ErrRequestBytesExceed", + Description: "Simultaneous request bytes exceed limitations", + HTTPStatusCode: http.StatusTooManyRequests, + }, } // GetAPIError provides API Error for input API error code. diff --git a/weed/s3api/stats.go b/weed/s3api/stats.go index b667b32a0..003807a25 100644 --- a/weed/s3api/stats.go +++ b/weed/s3api/stats.go @@ -1,8 +1,8 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" stats_collect "github.com/chrislusf/seaweedfs/weed/stats" - "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strconv" "time" @@ -28,11 +28,12 @@ func (r *StatusRecorder) Flush() { func track(f http.HandlerFunc, action string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Server", "SeaweedFS S3 "+util.VERSION) + bucket, _ := s3_constants.GetBucketAndObject(r) + w.Header().Set("Server", "SeaweedFS S3") recorder := NewStatusResponseWriter(w) start := time.Now() f(recorder, r) - stats_collect.S3RequestHistogram.WithLabelValues(action).Observe(time.Since(start).Seconds()) - stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status)).Inc() + stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds()) + stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc() } } diff --git a/weed/s3api/tags.go b/weed/s3api/tags.go index 9ff7d1fba..979e5a80c 100644 --- a/weed/s3api/tags.go +++ b/weed/s3api/tags.go @@ -14,8 +14,9 @@ type TagSet struct { } type Tagging struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + XMLName xml.Name `xml:"Tagging"` TagSet TagSet `xml:"TagSet"` + Xmlns string `xml:"xmlns,attr"` } func (t *Tagging) ToTags() map[string]string { @@ -27,7 +28,7 @@ func (t *Tagging) ToTags() map[string]string { } func FromTags(tags map[string]string) (t *Tagging) { - t = &Tagging{} + t = &Tagging{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/"} for k, v := range tags { t.TagSet.Tag = append(t.TagSet.Tag, Tag{ Key: k, diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index 52adb36c1..d8beb1922 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -32,6 +32,7 @@ func TestXMLUnmarshall(t *testing.T) { func TestXMLMarshall(t *testing.T) { tags := &Tagging{ + Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", TagSet: TagSet{ []Tag{ { diff --git a/weed/security/guard.go b/weed/security/guard.go index 87ec91ec1..8cb52620e 100644 --- a/weed/security/guard.go +++ b/weed/security/guard.go @@ -123,5 +123,5 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { } glog.V(0).Infof("Not in whitelist: %s", r.RemoteAddr) - return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr) + return fmt.Errorf("Not in whitelist: %s", r.RemoteAddr) } diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 7327f7b8b..82ba0df12 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -13,12 +13,21 @@ import ( type EncodedJwt string type SigningKey []byte +// SeaweedFileIdClaims is created by Master server(s) and consumed by Volume server(s), +// restricting the access this JWT allows to only a single file. type SeaweedFileIdClaims struct { Fid string `json:"fid"` jwt.StandardClaims } -func GenJwt(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { +// SeaweedFilerClaims is created e.g. by S3 proxy server and consumed by Filer server. +// Right now, it only contains the standard claims; but this might be extended later +// for more fine-grained permissions. +type SeaweedFilerClaims struct { + jwt.StandardClaims +} + +func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { if len(signingKey) == 0 { return "" } @@ -39,6 +48,28 @@ func GenJwt(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJw return EncodedJwt(encoded) } +// GenJwtForFilerServer creates a JSON-web-token for using the authenticated Filer API. Used f.e. inside +// the S3 API +func GenJwtForFilerServer(signingKey SigningKey, expiresAfterSec int) EncodedJwt { + if len(signingKey) == 0 { + return "" + } + + claims := SeaweedFilerClaims{ + jwt.StandardClaims{}, + } + if expiresAfterSec > 0 { + claims.ExpiresAt = time.Now().Add(time.Second * time.Duration(expiresAfterSec)).Unix() + } + t := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + encoded, e := t.SignedString([]byte(signingKey)) + if e != nil { + glog.V(0).Infof("Failed to sign claims %+v: %v", t.Claims, e) + return "" + } + return EncodedJwt(encoded) +} + func GetJwt(r *http.Request) EncodedJwt { // Get token from query params @@ -55,9 +86,9 @@ func GetJwt(r *http.Request) EncodedJwt { return EncodedJwt(tokenStr) } -func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt) (token *jwt.Token, err error) { +func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims) (token *jwt.Token, err error) { // check exp, nbf - return jwt.ParseWithClaims(string(tokenString), &SeaweedFileIdClaims{}, func(token *jwt.Token) (interface{}, error) { + return jwt.ParseWithClaims(string(tokenString), claims, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("unknown token method") } diff --git a/weed/security/tls.go b/weed/security/tls.go index 7d3ffcdca..137b7d06a 100644 --- a/weed/security/tls.go +++ b/weed/security/tls.go @@ -1,23 +1,22 @@ package security import ( - "context" "crypto/tls" "crypto/x509" - "github.com/chrislusf/seaweedfs/weed/util" - grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/peer" - "google.golang.org/grpc/status" + "fmt" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/security/advancedtls" "io/ioutil" "strings" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" + "time" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" ) +const credRefreshingInterval = time.Duration(5) * time.Hour + type Authenticator struct { AllowedWildcardDomain string AllowedCommonNames map[string]bool @@ -28,28 +27,39 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption return nil, nil } - // load cert/key, ca cert - cert, err := tls.LoadX509KeyPair(config.GetString(component+".cert"), config.GetString(component+".key")) + serverOptions := pemfile.Options{ + CertFile: config.GetString(component + ".cert"), + KeyFile: config.GetString(component + ".key"), + RefreshDuration: credRefreshingInterval, + } + + serverIdentityProvider, err := pemfile.NewProvider(serverOptions) if err != nil { - glog.V(1).Infof("load cert: %s / key: %s error: %v", - config.GetString(component+".cert"), - config.GetString(component+".key"), - err) + glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverOptions, err) return nil, nil } - caCert, err := ioutil.ReadFile(config.GetString("grpc.ca")) + + serverRootOptions := pemfile.Options{ + RootFile: config.GetString("grpc.ca"), + RefreshDuration: credRefreshingInterval, + } + serverRootProvider, err := pemfile.NewProvider(serverRootOptions) if err != nil { - glog.V(1).Infof("read ca cert file %s error: %v", config.GetString("grpc.ca"), err) + glog.Warningf("pemfile.NewProvider(%v) failed: %v", serverRootOptions, err) return nil, nil } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - ta := credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{cert}, - ClientCAs: caCertPool, - ClientAuth: tls.RequireAndVerifyClientCert, - }) + // Start a server and create a client using advancedtls API with Provider. + options := &advancedtls.ServerOptions{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: serverIdentityProvider, + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: serverRootProvider, + }, + RequireClientCert: true, + VType: advancedtls.CertVerification, + } allowedCommonNames := config.GetString(component + ".allowed_commonNames") allowedWildcardDomain := config.GetString("grpc.allowed_wildcard_domain") if allowedCommonNames != "" || allowedWildcardDomain != "" { @@ -61,7 +71,16 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption AllowedCommonNames: allowedCommonNamesMap, AllowedWildcardDomain: allowedWildcardDomain, } - return grpc.Creds(ta), grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(auther.Authenticate)) + options.VerifyPeer = auther.Authenticate + } else { + options.VerifyPeer = func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + return &advancedtls.VerificationResults{}, nil + } + } + ta, err := advancedtls.NewServerCreds(options) + if err != nil { + glog.Warningf("advancedtls.NewServerCreds(%v) failed: %v", options, err) + return nil, nil } return grpc.Creds(ta), nil } @@ -76,49 +95,70 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption { return grpc.WithInsecure() } - // load cert/key, cacert - cert, err := tls.LoadX509KeyPair(certFileName, keyFileName) + clientOptions := pemfile.Options{ + CertFile: certFileName, + KeyFile: keyFileName, + RefreshDuration: credRefreshingInterval, + } + clientProvider, err := pemfile.NewProvider(clientOptions) if err != nil { - glog.V(1).Infof("load cert/key error: %v", err) + glog.Warningf("pemfile.NewProvider(%v) failed %v", clientOptions, err) return grpc.WithInsecure() } - caCert, err := ioutil.ReadFile(caFileName) + clientRootOptions := pemfile.Options{ + RootFile: config.GetString("grpc.ca"), + RefreshDuration: credRefreshingInterval, + } + clientRootProvider, err := pemfile.NewProvider(clientRootOptions) if err != nil { - glog.V(1).Infof("read ca cert file error: %v", err) + glog.Warningf("pemfile.NewProvider(%v) failed: %v", clientRootOptions, err) + return grpc.WithInsecure() + } + options := &advancedtls.ClientOptions{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: clientProvider, + }, + VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + return &advancedtls.VerificationResults{}, nil + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: clientRootProvider, + }, + VType: advancedtls.CertVerification, + } + ta, err := advancedtls.NewClientCreds(options) + if err != nil { + glog.Warningf("advancedtls.NewClientCreds(%v) failed: %v", options, err) return grpc.WithInsecure() } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - ta := credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: caCertPool, - InsecureSkipVerify: true, - }) return grpc.WithTransportCredentials(ta) } -func (a Authenticator) Authenticate(ctx context.Context) (newCtx context.Context, err error) { - p, ok := peer.FromContext(ctx) - if !ok { - return ctx, status.Error(codes.Unauthenticated, "no peer found") +func LoadClientTLSHTTP(clientCertFile string) *tls.Config { + clientCerts, err := ioutil.ReadFile(clientCertFile) + if err != nil { + glog.Fatal(err) } - - tlsAuth, ok := p.AuthInfo.(credentials.TLSInfo) + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(clientCerts) if !ok { - return ctx, status.Error(codes.Unauthenticated, "unexpected peer transport credentials") + glog.Fatalf("Error processing client certificate in %s\n", clientCertFile) } - if len(tlsAuth.State.VerifiedChains) == 0 || len(tlsAuth.State.VerifiedChains[0]) == 0 { - return ctx, status.Error(codes.Unauthenticated, "could not verify peer certificate") + + return &tls.Config{ + ClientCAs: certPool, + ClientAuth: tls.RequireAndVerifyClientCert, } +} - commonName := tlsAuth.State.VerifiedChains[0][0].Subject.CommonName - if a.AllowedWildcardDomain != "" && strings.HasSuffix(commonName, a.AllowedWildcardDomain) { - return ctx, nil +func (a Authenticator) Authenticate(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + if a.AllowedWildcardDomain != "" && strings.HasSuffix(params.Leaf.Subject.CommonName, a.AllowedWildcardDomain) { + return &advancedtls.VerificationResults{}, nil } - if _, ok := a.AllowedCommonNames[commonName]; ok { - return ctx, nil + if _, ok := a.AllowedCommonNames[params.Leaf.Subject.CommonName]; ok { + return &advancedtls.VerificationResults{}, nil } - - return ctx, status.Errorf(codes.Unauthenticated, "invalid subject common name: %s", commonName) + err := fmt.Errorf("Authenticate: invalid subject client common name: %s", params.Leaf.Subject.CommonName) + glog.Error(err) + return nil, err } diff --git a/weed/sequence/etcd_sequencer.go b/weed/sequence/etcd_sequencer.go deleted file mode 100644 index 1fc378640..000000000 --- a/weed/sequence/etcd_sequencer.go +++ /dev/null @@ -1,296 +0,0 @@ -package sequence - -/* -Note : -(1) store the sequence in the ETCD cluster, and local file(sequence.dat) -(2) batch get the sequences from ETCD cluster, and store the max sequence id in the local file -(3) the sequence range is : [currentSeqId, maxSeqId), when the currentSeqId >= maxSeqId, fetch the new maxSeqId. -*/ - -import ( - "context" - "fmt" - "sync" - "time" - - "io" - "os" - "strconv" - "strings" - - "github.com/chrislusf/seaweedfs/weed/glog" - "go.etcd.io/etcd/client" -) - -const ( - // EtcdKeyPrefix = "/seaweedfs" - EtcdKeySequence = "/master/sequence" - EtcdContextTimeoutSecond = 100 * time.Second - DefaultEtcdSteps uint64 = 500 // internal counter - SequencerFileName = "sequencer.dat" - FileMaxSequenceLength = 128 -) - -type EtcdSequencer struct { - sequenceLock sync.Mutex - - // available sequence range : [currentSeqId, maxSeqId) - currentSeqId uint64 - maxSeqId uint64 - - keysAPI client.KeysAPI - seqFile *os.File -} - -func NewEtcdSequencer(etcdUrls string, metaFolder string) (*EtcdSequencer, error) { - file, err := openSequenceFile(metaFolder + "/" + SequencerFileName) - if nil != err { - return nil, fmt.Errorf("open sequence file fialed, %v", err) - } - - cli, err := client.New(client.Config{ - Endpoints: strings.Split(etcdUrls, ","), - Username: "", - Password: "", - }) - if err != nil { - return nil, err - } - keysApi := client.NewKeysAPI(cli) - - // TODO: the current sequence id in local file is not used - maxValue, _, err := readSequenceFile(file) - if err != nil { - return nil, fmt.Errorf("read sequence from file failed, %v", err) - } - glog.V(4).Infof("read sequence from file : %d", maxValue) - - newSeq, err := setMaxSequenceToEtcd(keysApi, maxValue) - if err != nil { - return nil, err - } - - sequencer := &EtcdSequencer{maxSeqId: newSeq, - currentSeqId: newSeq, - keysAPI: keysApi, - seqFile: file, - } - return sequencer, nil -} - -func (es *EtcdSequencer) NextFileId(count uint64) uint64 { - es.sequenceLock.Lock() - defer es.sequenceLock.Unlock() - - if (es.currentSeqId + count) >= es.maxSeqId { - reqSteps := DefaultEtcdSteps - if count > DefaultEtcdSteps { - reqSteps += count - } - maxId, err := batchGetSequenceFromEtcd(es.keysAPI, reqSteps) - glog.V(4).Infof("get max sequence id from etcd, %d", maxId) - if err != nil { - glog.Error(err) - return 0 - } - es.currentSeqId, es.maxSeqId = maxId-reqSteps, maxId - glog.V(4).Infof("current id : %d, max id : %d", es.currentSeqId, es.maxSeqId) - - if err := writeSequenceFile(es.seqFile, es.maxSeqId, es.currentSeqId); err != nil { - glog.Errorf("flush sequence to file failed, %v", err) - } - } - - ret := es.currentSeqId - es.currentSeqId += count - return ret -} - -/** -instead of collecting the max value from volume server, -the max value should be saved in local config file and ETCD cluster -*/ -func (es *EtcdSequencer) SetMax(seenValue uint64) { - es.sequenceLock.Lock() - defer es.sequenceLock.Unlock() - if seenValue > es.maxSeqId { - maxId, err := setMaxSequenceToEtcd(es.keysAPI, seenValue) - if err != nil { - glog.Errorf("set Etcd Max sequence failed : %v", err) - return - } - es.currentSeqId, es.maxSeqId = maxId, maxId - - if err := writeSequenceFile(es.seqFile, maxId, maxId); err != nil { - glog.Errorf("flush sequence to file failed, %v", err) - } - } -} - -func (es *EtcdSequencer) GetMax() uint64 { - return es.maxSeqId -} - -func (es *EtcdSequencer) Peek() uint64 { - return es.currentSeqId -} - -func batchGetSequenceFromEtcd(kvApi client.KeysAPI, step uint64) (uint64, error) { - if step <= 0 { - return 0, fmt.Errorf("the step must be large than 1") - } - - ctx, cancel := context.WithTimeout(context.Background(), EtcdContextTimeoutSecond) - var endSeqValue uint64 = 0 - defer cancel() - for { - getResp, err := kvApi.Get(ctx, EtcdKeySequence, &client.GetOptions{Recursive: false, Quorum: true}) - if err != nil { - return 0, err - } - if getResp.Node == nil { - continue - } - - prevValue := getResp.Node.Value - prevSeqValue, err := strconv.ParseUint(prevValue, 10, 64) - if err != nil { - return 0, fmt.Errorf("get sequence from etcd failed, %v", err) - } - endSeqValue = prevSeqValue + step - endSeqStr := strconv.FormatUint(endSeqValue, 10) - - _, err = kvApi.Set(ctx, EtcdKeySequence, endSeqStr, &client.SetOptions{PrevValue: prevValue}) - if err == nil { - break - } - glog.Error(err) - } - - return endSeqValue, nil -} - -/** -update the value of the key EtcdKeySequence in ETCD cluster with the parameter of maxSeq, -when the value of the key EtcdKeySequence is equal to or large than the parameter maxSeq, -return the value of EtcdKeySequence in the ETCD cluster; -when the value of the EtcdKeySequence is less than the parameter maxSeq, -return the value of the parameter maxSeq -*/ -func setMaxSequenceToEtcd(kvApi client.KeysAPI, maxSeq uint64) (uint64, error) { - maxSeqStr := strconv.FormatUint(maxSeq, 10) - ctx, cancel := context.WithTimeout(context.Background(), EtcdContextTimeoutSecond) - defer cancel() - - for { - getResp, err := kvApi.Get(ctx, EtcdKeySequence, &client.GetOptions{Recursive: false, Quorum: true}) - if err != nil { - if ce, ok := err.(client.Error); ok && (ce.Code == client.ErrorCodeKeyNotFound) { - _, err := kvApi.Create(ctx, EtcdKeySequence, maxSeqStr) - if err == nil { - continue - } - if ce, ok = err.(client.Error); ok && (ce.Code == client.ErrorCodeNodeExist) { - continue - } - return 0, err - } else { - return 0, err - } - } - - if getResp.Node == nil { - continue - } - prevSeqStr := getResp.Node.Value - prevSeq, err := strconv.ParseUint(prevSeqStr, 10, 64) - if err != nil { - return 0, err - } - if prevSeq >= maxSeq { - return prevSeq, nil - } - - _, err = kvApi.Set(ctx, EtcdKeySequence, maxSeqStr, &client.SetOptions{PrevValue: prevSeqStr}) - if err != nil { - return 0, err - } - } -} - -func openSequenceFile(file string) (*os.File, error) { - _, err := os.Stat(file) - if os.IsNotExist(err) { - fid, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return nil, err - } - if err := writeSequenceFile(fid, 1, 0); err != nil { - return nil, err - } - return fid, nil - } else { - return os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644) - } -} - -/* -read sequence and step from sequence file -*/ -func readSequenceFile(file *os.File) (uint64, uint64, error) { - sequence := make([]byte, FileMaxSequenceLength) - size, err := file.ReadAt(sequence, 0) - if (err != nil) && (err != io.EOF) { - err := fmt.Errorf("cannot read file %s, %v", file.Name(), err) - return 0, 0, err - } - sequence = sequence[0:size] - seqs := strings.Split(string(sequence), ":") - maxId, err := strconv.ParseUint(seqs[0], 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("parse sequence from file failed, %v", err) - } - - if len(seqs) > 1 { - step, err := strconv.ParseUint(seqs[1], 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("parse sequence from file failed, %v", err) - } - return maxId, step, nil - } - - return maxId, 0, nil -} - -/** -write the sequence and step to sequence file -*/ -func writeSequenceFile(file *os.File, sequence, step uint64) error { - _ = step - seqStr := fmt.Sprintf("%d:%d", sequence, sequence) - if _, err := file.Seek(0, 0); err != nil { - err = fmt.Errorf("cannot seek to the beginning of %s: %v", file.Name(), err) - return err - } - if err := file.Truncate(0); err != nil { - return fmt.Errorf("truncate sequence file faield : %v", err) - } - if _, err := file.WriteString(seqStr); err != nil { - return fmt.Errorf("write file %s failed, %v", file.Name(), err) - } - if err := file.Sync(); err != nil { - return fmt.Errorf("flush file %s failed, %v", file.Name(), err) - } - return nil -} - -// the UT helper method -// func deleteEtcdKey(kvApi client.KeysAPI, key string) error { -// ctx, cancel := context.WithTimeout(context.Background(), EtcdContextTimeoutSecond) -// defer cancel() -// _, err := kvApi.Delete(ctx, key, &client.DeleteOptions{Dir: false}) -// if err != nil { -// return err -// } -// return nil -// } diff --git a/weed/sequence/snowflake_sequencer_test.go b/weed/sequence/snowflake_sequencer_test.go new file mode 100644 index 000000000..731e330c5 --- /dev/null +++ b/weed/sequence/snowflake_sequencer_test.go @@ -0,0 +1,25 @@ +package sequence + +import ( + "encoding/hex" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSequencer(t *testing.T) { + seq, err := NewSnowflakeSequencer("for_test", 1) + assert.Equal(t, nil, err) + last := uint64(0) + bytes := make([]byte, types.NeedleIdSize) + for i := 0; i < 100; i++ { + next := seq.NextFileId(1) + types.NeedleIdToBytes(bytes, types.NeedleId(next)) + println(hex.EncodeToString(bytes)) + if last == next { + t.Errorf("last %d next %d", last, next) + } + last = next + } + +} diff --git a/weed/server/common.go b/weed/server/common.go index ad3842190..f02ec67ac 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -1,14 +1,17 @@ package weed_server import ( + "bufio" "bytes" "encoding/json" "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "io" "io/fs" "mime/multipart" "net/http" + "net/url" "path/filepath" "strconv" "strings" @@ -161,7 +164,16 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope } debug("upload file to store", url) - uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth) + uploadOption := &operation.UploadOption{ + UploadUrl: url, + Filename: pu.FileName, + Cipher: false, + IsInputCompressed: pu.IsGzipped, + MimeType: pu.MimeType, + PairMap: pu.PairMap, + Jwt: assignResult.Auth, + } + uploadResult, err := operation.UploadData(pu.Data, uploadOption) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return @@ -240,8 +252,20 @@ func handleStaticResources2(r *mux.Router) { r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS)))) } +func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) { + for header, values := range r.Header { + if normalizedHeader, ok := s3_constants.PassThroughHeaders[strings.ToLower(header)]; ok { + w.Header()[normalizedHeader] = values + } + } + adjustHeaderContentDisposition(w, r, filename) +} func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) { + if contentDisposition := w.Header().Get("Content-Disposition"); contentDisposition != "" { + return + } if filename != "" { + filename = url.QueryEscape(filename) contentDisposition := "inline" if r.FormValue("dl") != "" { if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl { @@ -254,10 +278,13 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) { rangeReq := r.Header.Get("Range") + bufferedWriter := bufio.NewWriterSize(w, 128*1024) + defer bufferedWriter.Flush() if rangeReq == "" { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) - if err := writeFn(w, 0, totalSize); err != nil { + if err := writeFn(bufferedWriter, 0, totalSize); err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -268,6 +295,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 //mostly copy from src/pkg/net/http/fs.go ranges, err := parseRange(rangeReq, totalSize) if err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) return } @@ -298,8 +326,9 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 w.Header().Set("Content-Range", ra.contentRange(totalSize)) w.WriteHeader(http.StatusPartialContent) - err = writeFn(w, ra.start, ra.length) + err = writeFn(bufferedWriter, ra.start, ra.length) if err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -338,7 +367,8 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) } w.WriteHeader(http.StatusPartialContent) - if _, err := io.CopyN(w, sendContent, sendSize); err != nil { + if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil { + glog.Errorf("processRangeRequest err: %v", err) http.Error(w, "Internal Error", http.StatusInternalServerError) return } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index e025e73dc..17d17c588 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -107,6 +107,7 @@ func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVol locs = append(locs, &filer_pb.Location{ Url: loc.Url, PublicUrl: loc.PublicUrl, + GrpcPort: uint32(loc.GrpcPort), }) } resp.LocationsMap[vidString] = &filer_pb.Locations{ @@ -143,10 +144,15 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr return &filer_pb.CreateEntryResponse{}, fmt.Errorf("CreateEntry cleanupChunks %s %s: %v", req.Directory, req.Entry.Name, err2) } + so, err := fs.detectStorageOption(string(util.NewFullPath(req.Directory, req.Entry.Name)), "", "", 0, "", "", "", "") + if err != nil { + return nil, err + } newEntry := filer.FromPbEntry(req.Directory, req.Entry) newEntry.Chunks = chunks + newEntry.TtlSec = so.TtlSeconds - createErr := fs.filer.CreateEntry(ctx, newEntry, req.OExcl, req.IsFromOtherCluster, req.Signatures) + createErr := fs.filer.CreateEntry(ctx, newEntry, req.OExcl, req.IsFromOtherCluster, req.Signatures, req.SkipCheckParentDirectory) if createErr == nil { fs.filer.DeleteChunks(garbage) @@ -210,10 +216,11 @@ func (fs *FilerServer) cleanupChunks(fullpath string, existingEntry *filer.Entry if newEntry.Attributes != nil { so, _ := fs.detectStorageOption(fullpath, - newEntry.Attributes.Collection, - newEntry.Attributes.Replication, + "", + "", newEntry.Attributes.TtlSec, - newEntry.Attributes.DiskType, + "", + "", "", "", ) // ignore readonly error for capacity needed to manifestize @@ -257,7 +264,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo } entry.Chunks = append(entry.Chunks, req.Chunks...) - so, err := fs.detectStorageOption(string(fullpath), entry.Collection, entry.Replication, entry.TtlSec, entry.DiskType, "", "") + so, err := fs.detectStorageOption(string(fullpath), "", "", entry.TtlSec, "", "", "", "") if err != nil { glog.Warningf("detectStorageOption: %v", err) return &filer_pb.AppendToEntryResponse{}, err @@ -268,7 +275,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo glog.V(0).Infof("MaybeManifestize: %v", err) } - err = fs.filer.CreateEntry(context.Background(), entry, false, false, nil) + err = fs.filer.CreateEntry(context.Background(), entry, false, false, nil, false) return &filer_pb.AppendToEntryResponse{}, err } @@ -287,7 +294,7 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) { - so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack) + so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack, req.DataNode) if err != nil { glog.V(3).Infof("AssignVolume: %v", err) return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume: %v", err)}, nil @@ -306,10 +313,13 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol } return &filer_pb.AssignVolumeResponse{ - FileId: assignResult.Fid, - Count: int32(assignResult.Count), - Url: assignResult.Url, - PublicUrl: assignResult.PublicUrl, + FileId: assignResult.Fid, + Count: int32(assignResult.Count), + Location: &filer_pb.Location{ + Url: assignResult.Url, + PublicUrl: assignResult.PublicUrl, + GrpcPort: uint32(assignResult.GrpcPort), + }, Auth: string(assignResult.Auth), Collection: so.Collection, Replication: so.Replication, @@ -321,7 +331,7 @@ func (fs *FilerServer) CollectionList(ctx context.Context, req *filer_pb.Collect glog.V(4).Infof("CollectionList %v", req) resp = &filer_pb.CollectionListResponse{} - err = fs.filer.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = fs.filer.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { masterResp, err := client.CollectionList(context.Background(), &master_pb.CollectionListRequest{ IncludeNormalVolumes: req.IncludeNormalVolumes, IncludeEcVolumes: req.IncludeEcVolumes, @@ -342,7 +352,7 @@ func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.Delet glog.V(4).Infof("DeleteCollection %v", req) - err = fs.filer.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = fs.filer.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ Name: req.GetCollection(), }) @@ -351,128 +361,3 @@ func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.Delet return &filer_pb.DeleteCollectionResponse{}, err } - -func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) { - - var output *master_pb.StatisticsResponse - - err = fs.filer.MasterClient.WithClient(func(masterClient master_pb.SeaweedClient) error { - grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{ - Replication: req.Replication, - Collection: req.Collection, - Ttl: req.Ttl, - DiskType: req.DiskType, - }) - if grpcErr != nil { - return grpcErr - } - - output = grpcResponse - return nil - }) - - if err != nil { - return nil, err - } - - return &filer_pb.StatisticsResponse{ - TotalSize: output.TotalSize, - UsedSize: output.UsedSize, - FileCount: output.FileCount, - }, nil -} - -func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) { - - clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId")) - - t := &filer_pb.GetFilerConfigurationResponse{ - Masters: fs.option.Masters, - Collection: fs.option.Collection, - Replication: fs.option.DefaultReplication, - MaxMb: uint32(fs.option.MaxMB), - DirBuckets: fs.filer.DirBucketsPath, - Cipher: fs.filer.Cipher, - Signature: fs.filer.Signature, - MetricsAddress: fs.metricsAddress, - MetricsIntervalSec: int32(fs.metricsIntervalSec), - Version: util.Version(), - ClusterId: string(clusterId), - } - - glog.V(4).Infof("GetFilerConfiguration: %v", t) - - return t, nil -} - -func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error { - - req, err := stream.Recv() - if err != nil { - return err - } - - clientName := fmt.Sprintf("%s:%d", req.Name, req.GrpcPort) - m := make(map[string]bool) - for _, tp := range req.Resources { - m[tp] = true - } - fs.brokersLock.Lock() - fs.brokers[clientName] = m - glog.V(0).Infof("+ broker %v", clientName) - fs.brokersLock.Unlock() - - defer func() { - fs.brokersLock.Lock() - delete(fs.brokers, clientName) - glog.V(0).Infof("- broker %v: %v", clientName, err) - fs.brokersLock.Unlock() - }() - - for { - if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil { - glog.V(0).Infof("send broker %v: %+v", clientName, err) - return err - } - // println("replied") - - if _, err := stream.Recv(); err != nil { - glog.V(0).Infof("recv broker %v: %v", clientName, err) - return err - } - // println("received") - } - -} - -func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) { - - resp = &filer_pb.LocateBrokerResponse{} - - fs.brokersLock.Lock() - defer fs.brokersLock.Unlock() - - var localBrokers []*filer_pb.LocateBrokerResponse_Resource - - for b, m := range fs.brokers { - if _, found := m[req.Resource]; found { - resp.Found = true - resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{ - { - GrpcAddresses: b, - ResourceCount: int32(len(m)), - }, - } - return - } - localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{ - GrpcAddresses: b, - ResourceCount: int32(len(m)), - }) - } - - resp.Resources = localBrokers - - return resp, nil - -} diff --git a/weed/server/filer_grpc_server_admin.go b/weed/server/filer_grpc_server_admin.go new file mode 100644 index 000000000..df5b8fa1e --- /dev/null +++ b/weed/server/filer_grpc_server_admin.go @@ -0,0 +1,178 @@ +package weed_server + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "time" +) + +func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) { + + var output *master_pb.StatisticsResponse + + err = fs.filer.MasterClient.WithClient(false, func(masterClient master_pb.SeaweedClient) error { + grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{ + Replication: req.Replication, + Collection: req.Collection, + Ttl: req.Ttl, + DiskType: req.DiskType, + }) + if grpcErr != nil { + return grpcErr + } + + output = grpcResponse + return nil + }) + + if err != nil { + return nil, err + } + + return &filer_pb.StatisticsResponse{ + TotalSize: output.TotalSize, + UsedSize: output.UsedSize, + FileCount: output.FileCount, + }, nil +} + +func (fs *FilerServer) Ping(ctx context.Context, req *filer_pb.PingRequest) (resp *filer_pb.PingResponse, pingErr error) { + resp = &filer_pb.PingResponse{ + StartTimeNs: time.Now().UnixNano(), + } + if req.TargetType == cluster.FilerType { + pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.VolumeServerType { + pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.MasterType { + pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client master_pb.SeaweedClient) error { + pingResp, err := client.Ping(ctx, &master_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if pingErr != nil { + pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr) + } + resp.StopTimeNs = time.Now().UnixNano() + return +} + +func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) { + + clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId")) + + t := &filer_pb.GetFilerConfigurationResponse{ + Masters: pb.ToAddressStringsFromMap(fs.option.Masters), + Collection: fs.option.Collection, + Replication: fs.option.DefaultReplication, + MaxMb: uint32(fs.option.MaxMB), + DirBuckets: fs.filer.DirBucketsPath, + Cipher: fs.filer.Cipher, + Signature: fs.filer.Signature, + MetricsAddress: fs.metricsAddress, + MetricsIntervalSec: int32(fs.metricsIntervalSec), + Version: util.Version(), + ClusterId: string(clusterId), + FilerGroup: fs.option.FilerGroup, + } + + glog.V(4).Infof("GetFilerConfiguration: %v", t) + + return t, nil +} + +func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error { + + req, err := stream.Recv() + if err != nil { + return err + } + + clientName := util.JoinHostPort(req.Name, int(req.GrpcPort)) + m := make(map[string]bool) + for _, tp := range req.Resources { + m[tp] = true + } + fs.brokersLock.Lock() + fs.brokers[clientName] = m + glog.V(0).Infof("+ broker %v", clientName) + fs.brokersLock.Unlock() + + defer func() { + fs.brokersLock.Lock() + delete(fs.brokers, clientName) + glog.V(0).Infof("- broker %v: %v", clientName, err) + fs.brokersLock.Unlock() + }() + + for { + if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil { + glog.V(0).Infof("send broker %v: %+v", clientName, err) + return err + } + // println("replied") + + if _, err := stream.Recv(); err != nil { + glog.V(0).Infof("recv broker %v: %v", clientName, err) + return err + } + // println("received") + } + +} + +func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) { + + resp = &filer_pb.LocateBrokerResponse{} + + fs.brokersLock.Lock() + defer fs.brokersLock.Unlock() + + var localBrokers []*filer_pb.LocateBrokerResponse_Resource + + for b, m := range fs.brokers { + if _, found := m[req.Resource]; found { + resp.Found = true + resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{ + { + GrpcAddresses: b, + ResourceCount: int32(len(m)), + }, + } + return + } + localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{ + GrpcAddresses: b, + ResourceCount: int32(len(m)), + }) + } + + resp.Resources = localBrokers + + return resp, nil + +} diff --git a/weed/server/filer_grpc_server_remote.go b/weed/server/filer_grpc_server_remote.go index 8064431c5..3be986023 100644 --- a/weed/server/filer_grpc_server_remote.go +++ b/weed/server/filer_grpc_server_remote.go @@ -3,20 +3,22 @@ package weed_server import ( "context" "fmt" + "strings" + "sync" + "time" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/proto" - "strings" - "sync" - "time" ) -func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.DownloadToLocalRequest) (*filer_pb.DownloadToLocalResponse, error) { +func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) { // load all mappings mappingEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE)) @@ -56,14 +58,13 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo return nil, err } - resp := &filer_pb.DownloadToLocalResponse{} + resp := &filer_pb.CacheRemoteObjectToLocalClusterResponse{} if entry.Remote == nil || entry.Remote.RemoteSize == 0 { return resp, nil } // detect storage option - // replication level is set to "000" to ensure only need to ask one volume server to fetch the data. - so, err := fs.detectStorageOption(req.Directory, "", "000", 0, "", "", "") + so, err := fs.detectStorageOption(req.Directory, "", "", 0, "", "", "", "") if err != nil { return resp, err } @@ -111,14 +112,26 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo return } + var replicas []*volume_server_pb.FetchAndWriteNeedleRequest_Replica + for _, r := range assignResult.Replicas { + replicas = append(replicas, &volume_server_pb.FetchAndWriteNeedleRequest_Replica{ + Url: r.Url, + PublicUrl: r.PublicUrl, + GrpcPort: int32(r.GrpcPort), + }) + } + // tell filer to tell volume server to download into needles - err = operation.WithVolumeServerClient(assignResult.Url, fs.grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + assignedServerAddress := pb.NewServerAddressWithGrpcPort(assignResult.Url, assignResult.GrpcPort) + err = operation.WithVolumeServerClient(false, assignedServerAddress, fs.grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, fetchAndWriteErr := volumeServerClient.FetchAndWriteNeedle(context.Background(), &volume_server_pb.FetchAndWriteNeedleRequest{ - VolumeId: uint32(fileId.VolumeId), - NeedleId: uint64(fileId.Key), - Cookie: uint32(fileId.Cookie), - Offset: localOffset, - Size: size, + VolumeId: uint32(fileId.VolumeId), + NeedleId: uint64(fileId.Key), + Cookie: uint32(fileId.Cookie), + Offset: localOffset, + Size: size, + Replicas: replicas, + Auth: string(assignResult.Auth), RemoteConf: storageConf, RemoteLocation: &remote_pb.RemoteStorageLocation{ Name: remoteStorageMountedLocation.Name, @@ -166,6 +179,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo // this skips meta data log events if err := fs.filer.Store.UpdateEntry(context.Background(), newEntry); err != nil { + fs.filer.DeleteChunks(chunks) return nil, err } fs.filer.DeleteChunks(garbage) diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index 8a11c91e3..7d6650b53 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "time" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -33,7 +34,7 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom return nil, fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err) } - moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName, req.Signatures) + moveErr := fs.moveEntry(ctx, nil, oldParent, oldEntry, newParent, req.NewName, req.Signatures) if moveErr != nil { fs.filer.RollbackTransaction(ctx) return nil, fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, moveErr) @@ -47,11 +48,64 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom return &filer_pb.AtomicRenameEntryResponse{}, nil } -func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, signatures []int32) error { +func (fs *FilerServer) StreamRenameEntry(req *filer_pb.StreamRenameEntryRequest, stream filer_pb.SeaweedFiler_StreamRenameEntryServer) (err error) { - if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error { + glog.V(1).Infof("StreamRenameEntry %v", req) + + oldParent := util.FullPath(filepath.ToSlash(req.OldDirectory)) + newParent := util.FullPath(filepath.ToSlash(req.NewDirectory)) + + if err := fs.filer.CanRename(oldParent, newParent); err != nil { + return err + } + + ctx := context.Background() + + ctx, err = fs.filer.BeginTransaction(ctx) + if err != nil { + return err + } + + oldEntry, err := fs.filer.FindEntry(ctx, oldParent.Child(req.OldName)) + if err != nil { + fs.filer.RollbackTransaction(ctx) + return fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err) + } + + if oldEntry.IsDirectory() { + // follow https://pubs.opengroup.org/onlinepubs/000095399/functions/rename.html + targetDir := newParent.Child(req.NewName) + newEntry, err := fs.filer.FindEntry(ctx, targetDir) + if err == nil { + if !newEntry.IsDirectory() { + fs.filer.RollbackTransaction(ctx) + return fmt.Errorf("%s is not directory", targetDir) + } + if entries, _, _ := fs.filer.ListDirectoryEntries(context.Background(), targetDir, "", false, 1, "", "", ""); len(entries) > 0 { + return fmt.Errorf("%s is not empty", targetDir) + } + } + } + + moveErr := fs.moveEntry(ctx, stream, oldParent, oldEntry, newParent, req.NewName, req.Signatures) + if moveErr != nil { + fs.filer.RollbackTransaction(ctx) + return fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, moveErr) + } else { + if commitError := fs.filer.CommitTransaction(ctx); commitError != nil { + fs.filer.RollbackTransaction(ctx) + return fmt.Errorf("%s/%s move commit error: %v", req.OldDirectory, req.OldName, commitError) + } + } + + return nil +} + +func (fs *FilerServer) moveEntry(ctx context.Context, stream filer_pb.SeaweedFiler_StreamRenameEntryServer, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, signatures []int32) error { + + if err := fs.moveSelfEntry(ctx, stream, oldParent, entry, newParent, newName, func() error { if entry.IsDirectory() { - if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, signatures); err != nil { + if err := fs.moveFolderSubEntries(ctx, stream, oldParent, entry, newParent, newName, signatures); err != nil { return err } } @@ -63,7 +117,7 @@ func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, e return nil } -func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, signatures []int32) error { +func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, stream filer_pb.SeaweedFiler_StreamRenameEntryServer, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, signatures []int32) error { currentDirPath := oldParent.Child(entry.Name()) newDirPath := newParent.Child(newName) @@ -84,7 +138,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. for _, item := range entries { lastFileName = item.Name() // println("processing", lastFileName) - err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name(), signatures) + err := fs.moveEntry(ctx, stream, currentDirPath, item, newDirPath, item.Name(), signatures) if err != nil { return err } @@ -96,7 +150,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. return nil } -func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error, signatures []int32) error { +func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.SeaweedFiler_StreamRenameEntryServer, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error, signatures []int32) error { oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName) @@ -109,15 +163,37 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat // add to new directory newEntry := &filer.Entry{ - FullPath: newPath, - Attr: entry.Attr, - Chunks: entry.Chunks, - Extended: entry.Extended, - Content: entry.Content, + FullPath: newPath, + Attr: entry.Attr, + Chunks: entry.Chunks, + Extended: entry.Extended, + Content: entry.Content, + HardLinkCounter: entry.HardLinkCounter, + HardLinkId: entry.HardLinkId, + Remote: entry.Remote, + Quota: entry.Quota, } - if createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, signatures); createErr != nil { + if createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, signatures, false); createErr != nil { return createErr } + if stream != nil { + if err := stream.Send(&filer_pb.StreamRenameEntryResponse{ + Directory: string(oldParent), + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{ + Name: entry.Name(), + }, + NewEntry: newEntry.ToProtoEntry(), + DeleteChunks: false, + NewParentPath: string(newParent), + IsFromOtherCluster: false, + Signatures: nil, + }, + TsNs: time.Now().UnixNano(), + }); err != nil { + return err + } + } if moveFolderSubEntries != nil { if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil { @@ -130,6 +206,24 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat if deleteErr != nil { return deleteErr } + if stream != nil { + if err := stream.Send(&filer_pb.StreamRenameEntryResponse{ + Directory: string(oldParent), + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{ + Name: entry.Name(), + }, + NewEntry: nil, + DeleteChunks: false, + NewParentPath: "", + IsFromOtherCluster: false, + Signatures: nil, + }, + TsNs: time.Now().UnixNano(), + }); err != nil { + return err + } + } return nil diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index 3fdac1b26..da710234b 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -2,6 +2,7 @@ package weed_server import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/stats" "strings" "time" @@ -23,9 +24,11 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, peerAddress := findClientAddress(stream.Context(), 0) - clientName := fs.addClient(req.ClientName, peerAddress) - - defer fs.deleteClient(clientName) + alreadyKnown, clientName := fs.addClient(req.ClientName, peerAddress, req.ClientId) + if alreadyKnown { + return fmt.Errorf("duplicated subscription detected for client %s id %d", clientName, req.ClientId) + } + defer fs.deleteClient(clientName, req.ClientId) lastReadTime := time.Unix(0, req.SinceNs) glog.V(0).Infof(" %v starts to subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) @@ -37,28 +40,27 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, var processedTsNs int64 var readPersistedLogErr error var readInMemoryLogErr error + var isDone bool for { glog.V(4).Infof("read on disk %v aggregated subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) - processedTsNs, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) + processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, req.UntilNs, eachLogEntryFn) if readPersistedLogErr != nil { return fmt.Errorf("reading from persisted logs: %v", readPersistedLogErr) } + if isDone { + return nil + } if processedTsNs != 0 { lastReadTime = time.Unix(0, processedTsNs) - } else { - if readInMemoryLogErr == log_buffer.ResumeFromDiskError { - time.Sleep(1127 * time.Millisecond) - continue - } } glog.V(4).Infof("read in memory %v aggregated subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) - lastReadTime, readInMemoryLogErr = fs.filer.MetaAggregator.MetaLogBuffer.LoopProcessLogData("aggMeta:"+clientName, lastReadTime, func() bool { + lastReadTime, isDone, readInMemoryLogErr = fs.filer.MetaAggregator.MetaLogBuffer.LoopProcessLogData("aggMeta:"+clientName, lastReadTime, req.UntilNs, func() bool { fs.filer.MetaAggregator.ListenersLock.Lock() fs.filer.MetaAggregator.ListenersCond.Wait() fs.filer.MetaAggregator.ListenersLock.Unlock() @@ -73,6 +75,9 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, break } } + if isDone { + return nil + } time.Sleep(1127 * time.Millisecond) } @@ -85,9 +90,11 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq peerAddress := findClientAddress(stream.Context(), 0) - clientName := fs.addClient(req.ClientName, peerAddress) - - defer fs.deleteClient(clientName) + alreadyKnown, clientName := fs.addClient(req.ClientName, peerAddress, req.ClientId) + if alreadyKnown { + return fmt.Errorf("duplicated local subscription detected for client %s id %d", clientName, req.ClientId) + } + defer fs.deleteClient(clientName, req.ClientId) lastReadTime := time.Unix(0, req.SinceNs) glog.V(0).Infof(" %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) @@ -99,14 +106,19 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq var processedTsNs int64 var readPersistedLogErr error var readInMemoryLogErr error + var isDone bool for { // println("reading from persisted logs ...") glog.V(0).Infof("read on disk %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) - processedTsNs, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) + processedTsNs, isDone, readPersistedLogErr = fs.filer.ReadPersistedLogBuffer(lastReadTime, req.UntilNs, eachLogEntryFn) if readPersistedLogErr != nil { + glog.V(0).Infof("read on disk %v local subscribe %s from %+v: %v", clientName, req.PathPrefix, lastReadTime, readPersistedLogErr) return fmt.Errorf("reading from persisted logs: %v", readPersistedLogErr) } + if isDone { + return nil + } if processedTsNs != 0 { lastReadTime = time.Unix(0, processedTsNs) @@ -119,7 +131,7 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq glog.V(0).Infof("read in memory %v local subscribe %s from %+v", clientName, req.PathPrefix, lastReadTime) - lastReadTime, readInMemoryLogErr = fs.filer.LocalMetaLogBuffer.LoopProcessLogData("localMeta:"+clientName, lastReadTime, func() bool { + lastReadTime, isDone, readInMemoryLogErr = fs.filer.LocalMetaLogBuffer.LoopProcessLogData("localMeta:"+clientName, lastReadTime, req.UntilNs, func() bool { fs.listenersLock.Lock() fs.listenersCond.Wait() fs.listenersLock.Unlock() @@ -130,11 +142,13 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq continue } glog.Errorf("processed to %v: %v", lastReadTime, readInMemoryLogErr) - time.Sleep(1127 * time.Millisecond) if readInMemoryLogErr != log_buffer.ResumeError { break } } + if isDone { + return nil + } } return readInMemoryLogErr @@ -201,17 +215,24 @@ func (fs *FilerServer) eachEventNotificationFn(req *filer_pb.SubscribeMetadataRe return nil } - if !strings.HasPrefix(fullpath, req.PathPrefix) { - if eventNotification.NewParentPath != "" { - newFullPath := util.Join(eventNotification.NewParentPath, entryName) - if !strings.HasPrefix(newFullPath, req.PathPrefix) { + if hasPrefixIn(fullpath, req.PathPrefixes) { + // good + } else { + if !strings.HasPrefix(fullpath, req.PathPrefix) { + if eventNotification.NewParentPath != "" { + newFullPath := util.Join(eventNotification.NewParentPath, entryName) + if !strings.HasPrefix(newFullPath, req.PathPrefix) { + return nil + } + } else { return nil } - } else { - return nil } } + // collect timestamps for path + stats.FilerServerLastSendTsOfSubscribeGauge.WithLabelValues(fs.option.Host.String(), req.ClientName, req.PathPrefix).Set(float64(tsNs)) + message := &filer_pb.SubscribeMetadataResponse{ Directory: dirPath, EventNotification: eventNotification, @@ -227,12 +248,31 @@ func (fs *FilerServer) eachEventNotificationFn(req *filer_pb.SubscribeMetadataRe } } -func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string) { +func hasPrefixIn(text string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(text, p) { + return true + } + } + return false +} + +func (fs *FilerServer) addClient(clientType string, clientAddress string, clientId int32) (alreadyKnown bool, clientName string) { clientName = clientType + "@" + clientAddress glog.V(0).Infof("+ listener %v", clientName) + if clientId != 0 { + fs.knownListenersLock.Lock() + _, alreadyKnown = fs.knownListeners[clientId] + fs.knownListenersLock.Unlock() + } return } -func (fs *FilerServer) deleteClient(clientName string) { +func (fs *FilerServer) deleteClient(clientName string, clientId int32) { glog.V(0).Infof("- listener %v", clientName) + if clientId != 0 { + fs.knownListenersLock.Lock() + delete(fs.knownListeners, clientId) + fs.knownListenersLock.Unlock() + } } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 534bc4840..6bf0261ee 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -16,10 +16,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" @@ -34,7 +36,9 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/postgres2" _ "github.com/chrislusf/seaweedfs/weed/filer/redis" _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" + _ "github.com/chrislusf/seaweedfs/weed/filer/redis3" _ "github.com/chrislusf/seaweedfs/weed/filer/sqlite" + _ "github.com/chrislusf/seaweedfs/weed/filer/ydb" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/notification" _ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs" @@ -46,7 +50,8 @@ import ( ) type FilerOption struct { - Masters []string + Masters map[string]pb.ServerAddress + FilerGroup string Collection string DefaultReplication string DisableDirListing bool @@ -54,21 +59,23 @@ type FilerOption struct { DirListingLimit int DataCenter string Rack string + DataNode string DefaultLevelDbDir string DisableHttp bool - Host string - Port uint32 + Host pb.ServerAddress recursiveDelete bool Cipher bool SaveToFilerLimit int64 - Filers []string ConcurrentUploadLimit int64 + ShowUIDirectoryDelete bool } type FilerServer struct { + filer_pb.UnimplementedSeaweedFilerServer option *FilerOption secret security.SigningKey filer *filer.Filer + filerGuard *security.Guard grpcDialOption grpc.DialOption // metrics read from the master @@ -79,6 +86,10 @@ type FilerServer struct { listenersLock sync.Mutex listenersCond *sync.Cond + // track known metadata listeners + knownListenersLock sync.Mutex + knownListeners map[int32]struct{} + brokers map[string]map[string]bool brokersLock sync.Mutex @@ -88,9 +99,19 @@ type FilerServer struct { func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { + v := util.GetViper() + signingKey := v.GetString("jwt.filer_signing.key") + v.SetDefault("jwt.filer_signing.expires_after_seconds", 10) + expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds") + + readSigningKey := v.GetString("jwt.filer_signing.read.key") + v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) + readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + fs = &FilerServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), + knownListeners: make(map[int32]struct{}), brokers: make(map[string]map[string]bool), inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)), } @@ -100,20 +121,21 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) glog.Fatal("master list is required!") } - fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.Port, option.Collection, option.DefaultReplication, option.DataCenter, func() { + fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, func() { fs.listenersCond.Broadcast() }) fs.filer.Cipher = option.Cipher + // we do not support IP whitelist right now + fs.filerGuard = security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) fs.checkWithMaster() - go stats.LoopPushingMetric("filer", stats.SourceName(fs.option.Port), fs.metricsAddress, fs.metricsIntervalSec) - go fs.filer.KeepConnectedToMaster() + go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec) + go fs.filer.KeepMasterClientConnected() - v := util.GetViper() if !util.LoadConfiguration("filer", false) { - v.Set("leveldb2.enabled", true) - v.Set("leveldb2.dir", option.DefaultLevelDbDir) + v.SetDefault("leveldb2.enabled", true) + v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir) _, err := os.Stat(option.DefaultLevelDbDir) if os.IsNotExist(err) { os.MkdirAll(option.DefaultLevelDbDir, 0755) @@ -130,7 +152,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) // TODO deprecated, will be be removed after 2020-12-31 // replaced by https://github.com/chrislusf/seaweedfs/wiki/Path-Specific-Configuration // fs.filer.FsyncBuckets = v.GetStringSlice("filer.options.buckets_fsync") - fs.filer.LoadConfiguration(v) + isFresh := fs.filer.LoadConfiguration(v) notification.LoadConfiguration(v, "notification.") @@ -143,9 +165,15 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) readonlyMux.HandleFunc("/", fs.readonlyFilerHandler) } - fs.filer.AggregateFromPeers(fmt.Sprintf("%s:%d", option.Host, option.Port), option.Filers) - - fs.filer.LoadBuckets() + existingNodes := fs.filer.ListExistingPeerUpdates() + startFromTime := time.Now().Add(-filer.LogFlushInterval) + if isFresh { + glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes) + if err := fs.filer.MaybeBootstrapFromPeers(option.Host, existingNodes, startFromTime); err != nil { + glog.Fatalf("%s bootstrap from %+v", option.Host, existingNodes) + } + } + fs.filer.AggregateFromPeers(option.Host, existingNodes, startFromTime) fs.filer.LoadFilerConf() @@ -160,17 +188,10 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) func (fs *FilerServer) checkWithMaster() { - for _, master := range fs.option.Masters { - _, err := pb.ParseServerToGrpcAddress(master) - if err != nil { - glog.Fatalf("invalid master address %s: %v", master, err) - } - } - isConnected := false for !isConnected { for _, master := range fs.option.Masters { - readErr := operation.WithMasterServerClient(master, fs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + readErr := operation.WithMasterServerClient(false, master, fs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) if err != nil { return fmt.Errorf("get master %s configuration: %v", master, err) diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 118646a04..6f0d0b7ca 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -1,7 +1,9 @@ package weed_server import ( + "errors" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strings" @@ -15,6 +17,19 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { start := time.Now() + if r.Method == "OPTIONS" { + stats.FilerRequestCounter.WithLabelValues("options").Inc() + OptionsHandler(w, r, false) + stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) + return + } + + isReadHttpCall := r.Method == "GET" || r.Method == "HEAD" + if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) { + writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) + return + } + // proxy to volume servers var fileId string if strings.HasPrefix(r.RequestURI, "/?proxyChunkId=") { @@ -78,20 +93,31 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { fs.PostHandler(w, r, contentLength) stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) } - case "OPTIONS": - stats.FilerRequestCounter.WithLabelValues("options").Inc() - OptionsHandler(w, r, false) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) } } func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) { + + start := time.Now() + + // We handle OPTIONS first because it never should be authenticated + if r.Method == "OPTIONS" { + stats.FilerRequestCounter.WithLabelValues("options").Inc() + OptionsHandler(w, r, true) + stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) + return + } + + if !fs.maybeCheckJwtAuthorization(r, false) { + writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) + return + } + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) if r.Header.Get("Origin") != "" { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } - start := time.Now() switch r.Method { case "GET": stats.FilerRequestCounter.WithLabelValues("get").Inc() @@ -101,10 +127,6 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r) stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) - case "OPTIONS": - stats.FilerRequestCounter.WithLabelValues("options").Inc() - OptionsHandler(w, r, true) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) } } @@ -116,3 +138,41 @@ func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) { } w.Header().Add("Access-Control-Allow-Headers", "*") } + +// maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied +func (fs *FilerServer) maybeCheckJwtAuthorization(r *http.Request, isWrite bool) bool { + + var signingKey security.SigningKey + + if isWrite { + if len(fs.filerGuard.SigningKey) == 0 { + return true + } else { + signingKey = fs.filerGuard.SigningKey + } + } else { + if len(fs.filerGuard.ReadSigningKey) == 0 { + return true + } else { + signingKey = fs.filerGuard.ReadSigningKey + } + } + + tokenStr := security.GetJwt(r) + if tokenStr == "" { + glog.V(1).Infof("missing jwt from %s", r.RemoteAddr) + return false + } + + token, err := security.DecodeJwt(signingKey, tokenStr, &security.SeaweedFilerClaims{}) + if err != nil { + glog.V(1).Infof("jwt verification error from %s: %v", r.RemoteAddr, err) + return false + } + if !token.Valid { + glog.V(1).Infof("jwt invalid from %s: %v", r.RemoteAddr, tokenStr) + return false + } else { + return true + } +} diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go index b8b28790b..301d609ec 100644 --- a/weed/server/filer_server_handlers_proxy.go +++ b/weed/server/filer_server_handlers_proxy.go @@ -3,6 +3,7 @@ package weed_server import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/mem" "io" "math/rand" "net/http" @@ -62,6 +63,9 @@ func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Reques w.Header()[k] = v } w.WriteHeader(proxyResponse.StatusCode) - io.Copy(w, proxyResponse.Body) + + buf := mem.Allocate(128 * 1024) + defer mem.Free(buf) + io.CopyBuffer(w, proxyResponse.Body, buf) } diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 054a1bd00..28573f7b3 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/util/mem" "io" + "math" "mime" "net/http" - "net/url" "path/filepath" "strconv" "strings" @@ -17,11 +19,67 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) +// Validates the preconditions. Returns true if GET/HEAD operation should not proceed. +// Preconditions supported are: +// If-Modified-Since +// If-Unmodified-Since +// If-Match +// If-None-Match +func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Entry) bool { + + etag := filer.ETagEntry(entry) + /// When more than one conditional request header field is present in a + /// request, the order in which the fields are evaluated becomes + /// important. In practice, the fields defined in this document are + /// consistently implemented in a single, logical order, since "lost + /// update" preconditions have more strict requirements than cache + /// validation, a validated cache is more efficient than a partial + /// response, and entity tags are presumed to be more accurate than date + /// validators. https://tools.ietf.org/html/rfc7232#section-5 + if entry.Attr.Mtime.IsZero() { + return false + } + w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) + + ifMatchETagHeader := r.Header.Get("If-Match") + ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since") + if ifMatchETagHeader != "" { + if util.CanonicalizeETag(etag) != util.CanonicalizeETag(ifMatchETagHeader) { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + } else if ifUnmodifiedSinceHeader != "" { + if t, parseError := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); parseError == nil { + if t.Before(entry.Attr.Mtime) { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + } + } + + ifNoneMatchETagHeader := r.Header.Get("If-None-Match") + ifModifiedSinceHeader := r.Header.Get("If-Modified-Since") + if ifNoneMatchETagHeader != "" { + if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) { + w.WriteHeader(http.StatusNotModified) + return true + } + } else if ifModifiedSinceHeader != "" { + if t, parseError := time.Parse(http.TimeFormat, ifModifiedSinceHeader); parseError == nil { + if t.After(entry.Attr.Mtime) { + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + + return false +} + func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { path := r.URL.Path @@ -38,11 +96,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } if err == filer_pb.ErrNotFound { glog.V(1).Infof("Not found %s: %v", path, err) - stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc() w.WriteHeader(http.StatusNotFound) } else { glog.Errorf("Internal %s: %v", path, err) - stats.FilerRequestCounter.WithLabelValues("read.internalerror").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc() w.WriteHeader(http.StatusInternalServerError) } return @@ -62,10 +120,22 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } - // set etag + query := r.URL.Query() + if query.Get("metadata") == "true" { + if query.Get("resolveManifest") == "true" { + if entry.Chunks, _, err = filer.ResolveChunkManifest( + fs.filer.MasterClient.GetLookupFileIdFunction(), + entry.Chunks, 0, math.MaxInt64); err != nil { + err = fmt.Errorf("failed to resolve chunk manifest, err: %s", err.Error()) + writeJsonError(w, r, http.StatusInternalServerError, err) + } + } + writeJsonQuiet(w, r, http.StatusOK, entry) + return + } + etag := filer.ETagEntry(entry) - if ifm := r.Header.Get("If-Match"); ifm != "" && ifm != "\""+etag+"\"" { - w.WriteHeader(http.StatusPreconditionFailed) + if checkPreconditions(w, r, entry) { return } @@ -82,22 +152,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", mimeType) } - // if modified since - if !entry.Attr.Mtime.IsZero() { - w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) - if r.Header.Get("If-Modified-Since") != "" { - if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if !t.Before(entry.Attr.Mtime) { - w.WriteHeader(http.StatusNotModified) - return - } - } - } - } - // print out the header from extended properties for k, v := range entry.Extended { - w.Header().Set(k, string(v)) + if !strings.HasPrefix(k, "xattr-") { + // "xattr-" prefix is set in filesys.XATTR_PREFIX + w.Header().Set(k, string(v)) + } } //Seaweed custom header are not visible to Vue or javascript @@ -111,27 +171,20 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set("Access-Control-Expose-Headers", strings.Join(seaweedHeaders, ",")) //set tag count - if r.Method == "GET" { - tagCount := 0 - for k := range entry.Extended { - if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") { - tagCount++ - } - } - if tagCount > 0 { - w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) + tagCount := 0 + for k := range entry.Extended { + if strings.HasPrefix(k, s3_constants.AmzObjectTagging+"-") { + tagCount++ } } - - if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" { - w.WriteHeader(http.StatusNotModified) - return + if tagCount > 0 { + w.Header().Set(s3_constants.AmzTagCount, strconv.Itoa(tagCount)) } + setEtag(w, etag) filename := entry.Name() - filename = url.QueryEscape(filename) - adjustHeaderContentDisposition(w, r, filename) + adjustPassthroughHeaders(w, r, filename) totalSize := int64(entry.Size()) @@ -147,10 +200,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } width, height, mode, shouldResize := shouldResizeImages(ext, r) if shouldResize { - data, err := filer.ReadAll(fs.filer.MasterClient, entry.Chunks) + data := mem.Allocate(int(totalSize)) + defer mem.Free(data) + err := filer.ReadAll(data, fs.filer.MasterClient, entry.Chunks) if err != nil { glog.Errorf("failed to read %s: %v", path, err) - w.WriteHeader(http.StatusNotModified) + w.WriteHeader(http.StatusInternalServerError) return } rs, _, _ := images.Resized(ext, bytes.NewReader(data), width, height, mode) @@ -163,6 +218,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if offset+size <= int64(len(entry.Content)) { _, err := writer.Write(entry.Content[offset : offset+size]) if err != nil { + stats.FilerRequestCounter.WithLabelValues(stats.ErrorWriteEntry).Inc() glog.Errorf("failed to write entry content: %v", err) } return err @@ -170,10 +226,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) chunks := entry.Chunks if entry.IsInRemoteOnly() { dir, name := entry.FullPath.DirAndName() - if resp, err := fs.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{ + if resp, err := fs.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{ Directory: dir, Name: name, }); err != nil { + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadCache).Inc() + glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) return fmt.Errorf("cache %s: %v", entry.FullPath, err) } else { chunks = resp.Entry.Chunks @@ -182,6 +240,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size) if err != nil { + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadStream).Inc() glog.Errorf("failed to stream content %s: %v", r.URL, err) } return err diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 307c411b6..eaf17fa18 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -2,9 +2,6 @@ package weed_server import ( "context" - "encoding/base64" - "fmt" - "github.com/skip2/go-qrcode" "net/http" "strconv" "strings" @@ -49,8 +46,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque path = "" } + emptyFolder := true if len(entries) > 0 { lastFileName = entries[len(entries)-1].Name() + emptyFolder = false } glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries)) @@ -62,30 +61,27 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque Limit int LastFileName string ShouldDisplayLoadMore bool + EmptyFolder bool }{ path, entries, limit, lastFileName, shouldDisplayLoadMore, + emptyFolder, }) return } - var qrImageString string - img, err := qrcode.Encode(fmt.Sprintf("http://%s:%d%s", fs.option.Host, fs.option.Port, r.URL.Path), qrcode.Medium, 128) - if err == nil { - qrImageString = base64.StdEncoding.EncodeToString(img) - } - - ui.StatusTpl.Execute(w, struct { + err = ui.StatusTpl.Execute(w, struct { Path string Breadcrumbs []ui.Breadcrumb Entries interface{} Limit int LastFileName string ShouldDisplayLoadMore bool - QrImage string + EmptyFolder bool + ShowDirectoryDelete bool }{ path, ui.ToBreadcrumb(path), @@ -93,6 +89,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque limit, lastFileName, shouldDisplayLoadMore, - qrImageString, + emptyFolder, + fs.option.ShowUIDirectoryDelete, }) + if err != nil { + glog.V(0).Infof("Template Execute Error: %v", err) + } } diff --git a/weed/server/filer_server_handlers_tagging.go b/weed/server/filer_server_handlers_tagging.go index 70b5327d6..ae2093947 100644 --- a/weed/server/filer_server_handlers_tagging.go +++ b/weed/server/filer_server_handlers_tagging.go @@ -43,7 +43,7 @@ func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request) } } - if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil { + if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil, false); dbErr != nil { glog.V(0).Infof("failing to update %s tagging : %v", path, dbErr) writeJsonError(w, r, http.StatusInternalServerError, err) return @@ -82,7 +82,9 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque toDelete := strings.Split(r.URL.Query().Get("tagging"), ",") deletions := make(map[string]struct{}) for _, deletion := range toDelete { - deletions[deletion] = struct{}{} + if deletion != "" { + deletions[deletion] = struct{}{} + } } // delete all tags or specific tags @@ -107,7 +109,7 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque return } - if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil { + if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil, false); dbErr != nil { glog.V(0).Infof("failing to delete %s tagging : %v", path, dbErr) writeJsonError(w, r, http.StatusInternalServerError, err) return diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 39d983ab7..bbaf28aa8 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -3,6 +3,8 @@ package weed_server import ( "context" "errors" + "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "net/http" "os" "strings" @@ -57,14 +59,21 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte ctx := context.Background() + destination := r.RequestURI + if finalDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); finalDestination != "" { + destination = finalDestination + } + query := r.URL.Query() - so, err := fs.detectStorageOption0(r.RequestURI, + so, err := fs.detectStorageOption0(destination, query.Get("collection"), query.Get("replication"), query.Get("ttl"), query.Get("disk"), + query.Get("fsync"), query.Get("dataCenter"), query.Get("rack"), + query.Get("dataNode"), ) if err != nil { if err == ErrReadOnly { @@ -76,11 +85,78 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte return } - fs.autoChunk(ctx, w, r, contentLength, so) + if query.Has("mv.from") { + fs.move(ctx, w, r, so) + } else { + fs.autoChunk(ctx, w, r, contentLength, so) + } + util.CloseRequest(r) } +func (fs *FilerServer) move(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) { + src := r.URL.Query().Get("mv.from") + dst := r.URL.Path + + glog.V(2).Infof("FilerServer.move %v to %v", src, dst) + + var err error + if src, err = clearName(src); err != nil { + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + if dst, err = clearName(dst); err != nil { + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + src = strings.TrimRight(src, "/") + if src == "" { + err = fmt.Errorf("invalid source '/'") + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + + srcPath := util.FullPath(src) + dstPath := util.FullPath(dst) + srcEntry, err := fs.filer.FindEntry(ctx, srcPath) + if err != nil { + err = fmt.Errorf("failed to get src entry '%s', err: %s", src, err) + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + + oldDir, oldName := srcPath.DirAndName() + newDir, newName := dstPath.DirAndName() + newName = util.Nvl(newName, oldName) + + dstEntry, err := fs.filer.FindEntry(ctx, util.FullPath(strings.TrimRight(dst, "/"))) + if err != nil && err != filer_pb.ErrNotFound { + err = fmt.Errorf("failed to get dst entry '%s', err: %s", dst, err) + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + if err == nil && !dstEntry.IsDirectory() && srcEntry.IsDirectory() { + err = fmt.Errorf("move: cannot overwrite non-directory '%s' with directory '%s'", dst, src) + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + + _, err = fs.AtomicRenameEntry(ctx, &filer_pb.AtomicRenameEntryRequest{ + OldDirectory: oldDir, + OldName: oldName, + NewDirectory: newDir, + NewName: newName, + }) + if err != nil { + err = fmt.Errorf("failed to move entry from '%s' to '%s', err: %s", src, dst, err) + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + + w.WriteHeader(http.StatusNoContent) +} + // curl -X DELETE http://localhost:8888/path/to // curl -X DELETE http://localhost:8888/path/to?recursive=true // curl -X DELETE http://localhost:8888/path/to?recursive=true&ignoreRecursiveError=true @@ -115,7 +191,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication string, ttlSeconds int32, diskType, dataCenter, rack string) (*operation.StorageOption, error) { +func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication string, ttlSeconds int32, diskType, dataCenter, rack, dataNode string) (*operation.StorageOption, error) { rule := fs.filer.FilerConf.MatchStorageRule(requestURI) @@ -124,10 +200,9 @@ func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication } // required by buckets folder - bucketDefaultCollection, bucketDefaultReplication, fsync := "", "", false + bucketDefaultCollection := "" if strings.HasPrefix(requestURI, fs.filer.DirBucketsPath+"/") { bucketDefaultCollection = fs.filer.DetectBucket(util.FullPath(requestURI)) - bucketDefaultReplication, fsync = fs.filer.ReadBucketOption(bucketDefaultCollection) } if ttlSeconds == 0 { @@ -139,23 +214,33 @@ func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication } return &operation.StorageOption{ - Replication: util.Nvl(qReplication, rule.Replication, bucketDefaultReplication, fs.option.DefaultReplication), + Replication: util.Nvl(qReplication, rule.Replication, fs.option.DefaultReplication), Collection: util.Nvl(qCollection, rule.Collection, bucketDefaultCollection, fs.option.Collection), - DataCenter: util.Nvl(dataCenter, fs.option.DataCenter), - Rack: util.Nvl(rack, fs.option.Rack), + DataCenter: util.Nvl(dataCenter, rule.DataCenter, fs.option.DataCenter), + Rack: util.Nvl(rack, rule.Rack, fs.option.Rack), + DataNode: util.Nvl(dataNode, rule.DataNode, fs.option.DataNode), TtlSeconds: ttlSeconds, DiskType: util.Nvl(diskType, rule.DiskType), - Fsync: fsync || rule.Fsync, + Fsync: rule.Fsync, VolumeGrowthCount: rule.VolumeGrowthCount, }, nil } -func (fs *FilerServer) detectStorageOption0(requestURI, qCollection, qReplication string, qTtl string, diskType string, dataCenter, rack string) (*operation.StorageOption, error) { +func (fs *FilerServer) detectStorageOption0(requestURI, qCollection, qReplication string, qTtl string, diskType string, fsync string, dataCenter, rack, dataNode string) (*operation.StorageOption, error) { ttl, err := needle.ReadTTL(qTtl) if err != nil { glog.Errorf("fail to parse ttl %s: %v", qTtl, err) } - return fs.detectStorageOption(requestURI, qCollection, qReplication, int32(ttl.Minutes())*60, diskType, dataCenter, rack) + so, err := fs.detectStorageOption(requestURI, qCollection, qReplication, int32(ttl.Minutes())*60, diskType, dataCenter, rack, dataNode) + if so != nil { + if fsync == "false" { + so.Fsync = false + } else if fsync == "true" { + so.Fsync = true + } + } + + return so, err } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index a42e0fc97..9c2b9959f 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -3,6 +3,7 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "io" "net/http" "os" @@ -15,7 +16,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/util" @@ -62,7 +62,8 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * } } else if reply != nil { if len(md5bytes) > 0 { - w.Header().Set("Content-MD5", util.Base64Encode(md5bytes)) + md5InBase64 := util.Base64Encode(md5bytes) + w.Header().Set("Content-MD5", md5InBase64) } writeJsonQuiet(w, r, http.StatusCreated, reply) } @@ -96,6 +97,9 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite md5bytes = md5Hash.Sum(nil) filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent) + if replyerr != nil { + fs.filer.DeleteChunks(fileChunks) + } return } @@ -115,6 +119,9 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter md5bytes = md5Hash.Sum(nil) filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent) + if replyerr != nil { + fs.filer.DeleteChunks(fileChunks) + } return } @@ -123,6 +130,10 @@ func isAppend(r *http.Request) bool { return r.URL.Query().Get("op") == "append" } +func skipCheckParentDirEntry(r *http.Request) bool { + return r.URL.Query().Get("skipCheckParentDir") == "true" +} + func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { // detect file mode @@ -153,9 +164,13 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } var entry *filer.Entry + var newChunks []*filer_pb.FileChunk var mergedChunks []*filer_pb.FileChunk + + isAppend := isAppend(r) + isOffsetWrite := len(fileChunks) > 0 && fileChunks[0].Offset > 0 // when it is an append - if isAppend(r) { + if isAppend || isOffsetWrite { existingEntry, findErr := fs.filer.FindEntry(ctx, util.FullPath(path)) if findErr != nil && findErr != filer_pb.ErrNotFound { glog.V(0).Infof("failing to find %s: %v", path, findErr) @@ -166,11 +181,13 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa entry.Mtime = time.Now() entry.Md5 = nil // adjust chunk offsets - for _, chunk := range fileChunks { - chunk.Offset += int64(entry.FileSize) + if isAppend { + for _, chunk := range fileChunks { + chunk.Offset += int64(entry.FileSize) + } + entry.FileSize += uint64(chunkOffset) } - mergedChunks = append(entry.Chunks, fileChunks...) - entry.FileSize += uint64(chunkOffset) + newChunks = append(entry.Chunks, fileChunks...) // TODO if len(entry.Content) > 0 { @@ -180,27 +197,31 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } else { glog.V(4).Infoln("saving", path) - mergedChunks = fileChunks + newChunks = fileChunks entry = &filer.Entry{ FullPath: util.FullPath(path), Attr: filer.Attr{ - Mtime: time.Now(), - Crtime: time.Now(), - Mode: os.FileMode(mode), - Uid: OS_UID, - Gid: OS_GID, - Replication: so.Replication, - Collection: so.Collection, - TtlSec: so.TtlSeconds, - DiskType: so.DiskType, - Mime: contentType, - Md5: md5bytes, - FileSize: uint64(chunkOffset), + Mtime: time.Now(), + Crtime: time.Now(), + Mode: os.FileMode(mode), + Uid: OS_UID, + Gid: OS_GID, + TtlSec: so.TtlSeconds, + Mime: contentType, + Md5: md5bytes, + FileSize: uint64(chunkOffset), }, Content: content, } } + // maybe concatenate small chunks into one whole chunk + mergedChunks, replyerr = fs.maybeMergeChunks(so, newChunks) + if replyerr != nil { + glog.V(0).Infof("merge chunks %s: %v", r.RequestURI, replyerr) + mergedChunks = newChunks + } + // maybe compact entry chunks mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks) if replyerr != nil { @@ -208,6 +229,10 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa return } entry.Chunks = mergedChunks + if isOffsetWrite { + entry.Md5 = nil + entry.FileSize = entry.Size() + } filerResult = &FilerPostResult{ Name: fileName, @@ -217,13 +242,17 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa entry.Extended = SaveAmzMetaData(r, entry.Extended, false) for k, v := range r.Header { - if len(v) > 0 && (strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires") { - entry.Extended[k] = []byte(v[0]) + if len(v) > 0 && len(v[0]) > 0 { + if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { + entry.Extended[k] = []byte(v[0]) + } + if k == "Response-Content-Disposition" { + entry.Extended["Content-Disposition"] = []byte(v[0]) + } } } - if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil { - fs.filer.DeleteChunks(fileChunks) + if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r)); dbErr != nil { replyerr = dbErr filerResult.Error = dbErr.Error() glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr) @@ -241,7 +270,16 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs } // upload the chunk to the volume server - uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: urlLocation, + Filename: name, + Cipher: fs.option.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + } + uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption) if uploadErr != nil { return nil, "", "", uploadErr } @@ -291,7 +329,7 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http Name: util.FullPath(path).Name(), } - if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil { + if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, false); dbErr != nil { replyerr = dbErr filerResult.Error = dbErr.Error() glog.V(0).Infof("failing to create dir %s on filer server : %v", path, dbErr) @@ -308,21 +346,23 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool } } - if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { - metadata[xhttp.AmzStorageClass] = []byte(sc) + if sc := r.Header.Get(s3_constants.AmzStorageClass); sc != "" { + metadata[s3_constants.AmzStorageClass] = []byte(sc) } - if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" { + if tags := r.Header.Get(s3_constants.AmzObjectTagging); tags != "" { for _, v := range strings.Split(tags, "&") { tag := strings.Split(v, "=") if len(tag) == 2 { - metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) + metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) + } else if len(tag) == 1 { + metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = nil } } } for header, values := range r.Header { - if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) { + if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) { for _, value := range values { metadata[header] = []byte(value) } diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index acaa8f5ab..1f10d044e 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -44,7 +44,16 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht // println("detect2 mimetype to", pu.MimeType) } - uploadResult, uploadError := operation.UploadData(urlLocation, pu.FileName, true, uncompressedData, false, pu.MimeType, pu.PairMap, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: urlLocation, + Filename: pu.FileName, + Cipher: true, + IsInputCompressed: false, + MimeType: pu.MimeType, + PairMap: pu.PairMap, + Jwt: auth, + } + uploadResult, uploadError := operation.UploadData(uncompressedData, uploadOption) if uploadError != nil { return nil, fmt.Errorf("upload to volume server: %v", uploadError) } @@ -64,17 +73,14 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht entry := &filer.Entry{ FullPath: util.FullPath(path), Attr: filer.Attr{ - Mtime: time.Now(), - Crtime: time.Now(), - Mode: 0660, - Uid: OS_UID, - Gid: OS_GID, - Replication: so.Replication, - Collection: so.Collection, - TtlSec: so.TtlSeconds, - DiskType: so.DiskType, - Mime: pu.MimeType, - Md5: util.Base64Md5ToBytes(pu.ContentMd5), + Mtime: time.Now(), + Crtime: time.Now(), + Mode: 0660, + Uid: OS_UID, + Gid: OS_GID, + TtlSec: so.TtlSeconds, + Mime: pu.MimeType, + Md5: util.Base64Md5ToBytes(pu.ContentMd5), }, Chunks: fileChunks, } @@ -84,7 +90,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht Size: int64(pu.OriginalDataSize), } - if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil { + if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, false); dbErr != nil { fs.filer.DeleteChunks(entry.Chunks) err = dbErr filerResult.Error = dbErr.Error() diff --git a/weed/server/filer_server_handlers_write_merge.go b/weed/server/filer_server_handlers_write_merge.go new file mode 100644 index 000000000..dadc6f726 --- /dev/null +++ b/weed/server/filer_server_handlers_write_merge.go @@ -0,0 +1,11 @@ +package weed_server + +import ( + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func (fs *FilerServer) maybeMergeChunks(so *operation.StorageOption, inputChunks []*filer_pb.FileChunk) (mergedChunks []*filer_pb.FileChunk, err error) { + //TODO merge consecutive smaller chunks into a large chunk to reduce number of chunks + return inputChunks, nil +} diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index 2275ff1bc..fe3346402 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -3,11 +3,12 @@ package weed_server import ( "bytes" "crypto/md5" + "fmt" + "golang.org/x/exp/slices" "hash" "io" - "io/ioutil" "net/http" - "sort" + "strconv" "strings" "sync" "sync/atomic" @@ -29,9 +30,25 @@ var bufPool = sync.Pool{ } func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) { + query := r.URL.Query() + + isAppend := isAppend(r) + if query.Has("offset") { + offset := query.Get("offset") + offsetInt, err := strconv.ParseInt(offset, 10, 64) + if err != nil || offsetInt < 0 { + err = fmt.Errorf("invalid 'offset': '%s'", offset) + return nil, nil, 0, err, nil + } + if isAppend && offsetInt > 0 { + err = fmt.Errorf("cannot set offset when op=append") + return nil, nil, 0, err, nil + } + chunkOffset = offsetInt + } md5Hash = md5.New() - var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash)) + var partReader = io.NopCloser(io.TeeReader(reader, md5Hash)) var wg sync.WaitGroup var bytesBufferCounter int64 @@ -57,14 +74,15 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque dataSize, err := bytesBuffer.ReadFrom(limitedReader) - // data, err := ioutil.ReadAll(limitedReader) + // data, err := io.ReadAll(limitedReader) if err != nil || dataSize == 0 { bufPool.Put(bytesBuffer) atomic.AddInt64(&bytesBufferCounter, -1) bytesBufferLimitCond.Signal() + uploadErr = err break } - if chunkOffset == 0 && !isAppend(r) { + if chunkOffset == 0 && !isAppend { if dataSize < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) { chunkOffset += dataSize smallContent = make([]byte, dataSize) @@ -109,13 +127,12 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque wg.Wait() if uploadErr != nil { + fs.filer.DeleteChunks(fileChunks) return nil, md5Hash, 0, uploadErr, nil } - - sort.Slice(fileChunks, func(i, j int) bool { - return fileChunks[i].Offset < fileChunks[j].Offset + slices.SortFunc(fileChunks, func(a, b *filer_pb.FileChunk) bool { + return a.Offset < b.Offset }) - return fileChunks, md5Hash, chunkOffset, nil, smallContent } @@ -127,7 +144,16 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds()) }() - uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: urlLocation, + Filename: fileName, + Cipher: fs.option.Cipher, + IsInputCompressed: false, + MimeType: contentType, + PairMap: pairMap, + Jwt: auth, + } + uploadResult, err, data := operation.Upload(limitedReader, uploadOption) if uploadResult != nil && uploadResult.RetryCount > 0 { stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount)) } diff --git a/weed/server/filer_server_rocksdb.go b/weed/server/filer_server_rocksdb.go index 5fcc7e88f..75965e761 100644 --- a/weed/server/filer_server_rocksdb.go +++ b/weed/server/filer_server_rocksdb.go @@ -1,3 +1,4 @@ +//go:build rocksdb // +build rocksdb package weed_server diff --git a/weed/server/filer_ui/breadcrumb.go b/weed/server/filer_ui/breadcrumb.go index 5016117a8..3201ff76c 100644 --- a/weed/server/filer_ui/breadcrumb.go +++ b/weed/server/filer_ui/breadcrumb.go @@ -15,8 +15,12 @@ func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) { parts := strings.Split(fullpath, "/") for i := 0; i < len(parts); i++ { + name := parts[i] + if name == "" { + name = "/" + } crumb := Breadcrumb{ - Name: parts[i] + " /", + Name: name, Link: "/" + util.Join(parts[0:i+1]...), } if !strings.HasSuffix(crumb.Link, "/") { diff --git a/weed/server/filer_ui/filer.html b/weed/server/filer_ui/filer.html index 84dc4d4d6..c9d832e8f 100644 --- a/weed/server/filer_ui/filer.html +++ b/weed/server/filer_ui/filer.html @@ -11,6 +11,7 @@ #drop-area { border: 1px transparent; + margin-top: 5px; } #drop-area.highlight { @@ -26,6 +27,12 @@ border-radius: 2px; border: 1px solid #ccc; float: right; + margin-left: 2px; + margin-bottom: 0; + } + + label { + font-weight: normal; } .button:hover { @@ -36,10 +43,37 @@ display: none; } - .qrImage { + td, th { + vertical-align: bottom; + } + + .table-hover > tbody > tr:hover > * > div.operations { display: block; - margin-left: auto; - margin-right: auto; + } + + .table > tbody > tr { + height: 39px; + } + + div.operations { + display: none; + } + + .footer { + position: absolute; + bottom: 0px; + right: 5%; + min-width: 25%; + border-left: 1px solid #ccc; + border-right: 1px solid #ccc; + } + + .add-files { + font-size: 46px; + text-align: center; + border: 1px dashed #999; + padding-bottom: 9px; + margin: 0 2px; } @@ -53,12 +87,21 @@
+
+ + +
+
    {{ range $entry := .Breadcrumbs }} - +
  1. {{ $entry.Name }} - +
    2. {{ end }} - +
@@ -66,117 +109,250 @@
- - {{$path := .Path }} + {{ if .EmptyFolder }} +
+ + +
+ {{ else }} +
+ {{ $path := .Path }} + {{ $showDirDel := .ShowDirectoryDelete }} {{ range $entry_index, $entry := .Entries }} - + {{ end }} -
- {{if $entry.IsDirectory}} - + {{ if $entry.IsDirectory }} +   {{ $entry.Name }} - {{else}} + {{ else }} {{ $entry.Name }} - {{end}} + {{ end }} - {{if $entry.IsDirectory}} - {{else}} + {{ if not $entry.IsDirectory }} {{ $entry.Mime }}  - {{end}} + {{ end }} - {{if $entry.IsDirectory}} - {{else}} + {{ if not $entry.IsDirectory }} {{ $entry.Size | humanizeBytes }}  - {{end}} + {{ end }} + {{ $entry.Timestamp.Format "2006-01-02 15:04" }} +
+ + {{ if and $entry.IsDirectory $showDirDel }} + + {{ end }} + {{ if not $entry.IsDirectory }} + + {{ end }} +
+
+ {{ end }}
- {{if .ShouldDisplayLoadMore}} + {{ if .ShouldDisplayLoadMore }}
- - + -
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index afd479b21..4d0fbbc41 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -2,12 +2,17 @@ package weed_server import ( "context" + "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/backend" "net" - "strings" + "sort" "time" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/stats" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/raft" "google.golang.org/grpc/peer" @@ -17,16 +22,56 @@ import ( "github.com/chrislusf/seaweedfs/weed/topology" ) +func (ms *MasterServer) RegisterUuids(heartbeat *master_pb.Heartbeat) (duplicated_uuids []string, err error) { + ms.Topo.UuidAccessLock.Lock() + defer ms.Topo.UuidAccessLock.Unlock() + key := fmt.Sprintf("%s:%d", heartbeat.Ip, heartbeat.Port) + if ms.Topo.UuidMap == nil { + ms.Topo.UuidMap = make(map[string][]string) + } + // find whether new uuid exists + for k, v := range ms.Topo.UuidMap { + sort.Strings(v) + for _, id := range heartbeat.LocationUuids { + index := sort.SearchStrings(v, id) + if index < len(v) && v[index] == id { + duplicated_uuids = append(duplicated_uuids, id) + glog.Errorf("directory of %s on %s has been loaded", id, k) + } + } + } + if len(duplicated_uuids) > 0 { + return duplicated_uuids, errors.New("volume: Duplicated volume directories were loaded") + } + + ms.Topo.UuidMap[key] = heartbeat.LocationUuids + glog.V(0).Infof("found new uuid:%v %v , %v", key, heartbeat.LocationUuids, ms.Topo.UuidMap) + return nil, nil +} + +func (ms *MasterServer) UnRegisterUuids(ip string, port int) { + ms.Topo.UuidAccessLock.Lock() + defer ms.Topo.UuidAccessLock.Unlock() + key := fmt.Sprintf("%s:%d", ip, port) + delete(ms.Topo.UuidMap, key) + glog.V(0).Infof("remove volume server %v, online volume server: %v", key, ms.Topo.UuidMap) +} + func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error { var dn *topology.DataNode defer func() { if dn != nil { - + dn.Counter-- + if dn.Counter > 0 { + glog.V(0).Infof("disconnect phantom volume server %s:%d remaining %d", dn.Ip, dn.Port, dn.Counter) + return + } // if the volume server disconnects and reconnects quickly // the unregister and register can race with each other ms.Topo.UnRegisterDataNode(dn) glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port) + ms.UnRegisterUuids(dn.Ip, dn.Port) message := &master_pb.VolumeLocation{ Url: dn.Url(), @@ -40,13 +85,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } if len(message.DeletedVids) > 0 { - ms.clientChansLock.RLock() - for _, ch := range ms.clientChans { - ch <- message - } - ms.clientChansLock.RUnlock() + ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message}) } - } }() @@ -58,6 +98,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } else { glog.Warningf("SendHeartbeat.Recv: %v", err) } + stats.MasterReceivedHeartbeatCounter.WithLabelValues("error").Inc() return err } @@ -67,19 +108,34 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack) dc := ms.Topo.GetOrCreateDataCenter(dcName) rack := dc.GetOrCreateRack(rackName) - dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), heartbeat.PublicUrl, heartbeat.MaxVolumeCounts) - glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort()) + dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), int(heartbeat.GrpcPort), heartbeat.PublicUrl, heartbeat.MaxVolumeCounts) + glog.V(0).Infof("added volume server %d: %v:%d %v", dn.Counter, heartbeat.GetIp(), heartbeat.GetPort(), heartbeat.LocationUuids) + uuidlist, err := ms.RegisterUuids(heartbeat) + if err != nil { + if stream_err := stream.Send(&master_pb.HeartbeatResponse{ + DuplicatedUuids: uuidlist, + }); stream_err != nil { + glog.Warningf("SendHeartbeat.Send DuplicatedDirectory response to %s:%d %v", dn.Ip, dn.Port, stream_err) + return stream_err + } + return err + } + if err := stream.Send(&master_pb.HeartbeatResponse{ VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024, }); err != nil { glog.Warningf("SendHeartbeat.Send volume size to %s:%d %v", dn.Ip, dn.Port, err) return err } + stats.MasterReceivedHeartbeatCounter.WithLabelValues("dataNode").Inc() + dn.Counter++ } dn.AdjustMaxVolumeCounts(heartbeat.MaxVolumeCounts) glog.V(4).Infof("master received heartbeat %s", heartbeat.String()) + stats.MasterReceivedHeartbeatCounter.WithLabelValues("total").Inc() + var dataCenter string if dc := dn.GetDataCenter(); dc != nil { dataCenter = string(dc.Id()) @@ -89,6 +145,12 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ PublicUrl: dn.PublicUrl, DataCenter: dataCenter, } + if len(heartbeat.NewVolumes) > 0 { + stats.FilerRequestCounter.WithLabelValues("newVolumes").Inc() + } + if len(heartbeat.DeletedVolumes) > 0 { + stats.FilerRequestCounter.WithLabelValues("deletedVolumes").Inc() + } if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 { // process delta volume ids if exists for fast volume id updates for _, volInfo := range heartbeat.NewVolumes { @@ -102,7 +164,11 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes { + dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack) + ms.Topo.DataNodeRegistration(dcName, rackName, dn) + // process heartbeat.Volumes + stats.MasterReceivedHeartbeatCounter.WithLabelValues("Volumes").Inc() newVolumes, deletedVolumes := ms.Topo.SyncDataNodeRegistration(heartbeat.Volumes, dn) for _, v := range newVolumes { @@ -116,45 +182,41 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } if len(heartbeat.NewEcShards) > 0 || len(heartbeat.DeletedEcShards) > 0 { - + stats.MasterReceivedHeartbeatCounter.WithLabelValues("newEcShards").Inc() // update master internal volume layouts ms.Topo.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn) for _, s := range heartbeat.NewEcShards { - message.NewVids = append(message.NewVids, s.Id) + message.NewEcVids = append(message.NewEcVids, s.Id) } for _, s := range heartbeat.DeletedEcShards { - if dn.HasVolumesById(needle.VolumeId(s.Id)) { + if dn.HasEcShards(needle.VolumeId(s.Id)) { continue } - message.DeletedVids = append(message.DeletedVids, s.Id) + message.DeletedEcVids = append(message.DeletedEcVids, s.Id) } } if len(heartbeat.EcShards) > 0 || heartbeat.HasNoEcShards { - glog.V(1).Infof("master received ec shards from %s: %+v", dn.Url(), heartbeat.EcShards) + stats.MasterReceivedHeartbeatCounter.WithLabelValues("ecShards").Inc() + glog.V(4).Infof("master received ec shards from %s: %+v", dn.Url(), heartbeat.EcShards) newShards, deletedShards := ms.Topo.SyncDataNodeEcShards(heartbeat.EcShards, dn) // broadcast the ec vid changes to master clients for _, s := range newShards { - message.NewVids = append(message.NewVids, uint32(s.VolumeId)) + message.NewEcVids = append(message.NewEcVids, uint32(s.VolumeId)) } for _, s := range deletedShards { if dn.HasVolumesById(s.VolumeId) { continue } - message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId)) + message.DeletedEcVids = append(message.DeletedEcVids, uint32(s.VolumeId)) } } - if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 { - ms.clientChansLock.RLock() - for host, ch := range ms.clientChans { - glog.V(0).Infof("master send to %s: %s", host, message.String()) - ch <- message - } - ms.clientChansLock.RUnlock() + if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 || len(message.NewEcVids) > 0 || len(message.DeletedEcVids) > 0 { + ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message}) } // tell the volume servers about the leader @@ -164,7 +226,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ return err } if err := stream.Send(&master_pb.HeartbeatResponse{ - Leader: newLeader, + Leader: string(newLeader), }); err != nil { glog.Warningf("SendHeartbeat.Send response to to %s:%d %v", dn.Ip, dn.Port, err) return err @@ -185,17 +247,25 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ return ms.informNewLeader(stream) } - peerAddress := findClientAddress(stream.Context(), req.GrpcPort) + peerAddress := pb.ServerAddress(req.ClientAddress) // buffer by 1 so we don't end up getting stuck writing to stopChan forever stopChan := make(chan bool, 1) - clientName, messageChan := ms.addClient(req.Name, peerAddress) + clientName, messageChan := ms.addClient(req.FilerGroup, req.ClientType, peerAddress) + for _, update := range ms.Cluster.AddClusterNode(req.FilerGroup, req.ClientType, peerAddress, req.Version) { + ms.broadcastToClients(update) + } - defer ms.deleteClient(clientName) + defer func() { + for _, update := range ms.Cluster.RemoveClusterNode(req.FilerGroup, req.ClientType, peerAddress) { + ms.broadcastToClients(update) + } + ms.deleteClient(clientName) + }() for _, message := range ms.Topo.ToVolumeLocations() { - if sendErr := stream.Send(message); sendErr != nil { + if sendErr := stream.Send(&master_pb.KeepConnectedResponse{VolumeLocation: message}); sendErr != nil { return sendErr } } @@ -221,7 +291,10 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ } case <-ticker.C: if !ms.Topo.IsLeader() { + stats.MasterRaftIsleader.Set(0) return ms.informNewLeader(stream) + } else { + stats.MasterRaftIsleader.Set(1) } case <-stopChan: return nil @@ -230,22 +303,32 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ } +func (ms *MasterServer) broadcastToClients(message *master_pb.KeepConnectedResponse) { + ms.clientChansLock.RLock() + for _, ch := range ms.clientChans { + ch <- message + } + ms.clientChansLock.RUnlock() +} + func (ms *MasterServer) informNewLeader(stream master_pb.Seaweed_KeepConnectedServer) error { leader, err := ms.Topo.Leader() if err != nil { glog.Errorf("topo leader: %v", err) return raft.NotLeaderError } - if err := stream.Send(&master_pb.VolumeLocation{ - Leader: leader, + if err := stream.Send(&master_pb.KeepConnectedResponse{ + VolumeLocation: &master_pb.VolumeLocation{ + Leader: string(leader), + }, }); err != nil { return err } return nil } -func (ms *MasterServer) addClient(clientType string, clientAddress string) (clientName string, messageChan chan *master_pb.VolumeLocation) { - clientName = clientType + "@" + clientAddress +func (ms *MasterServer) addClient(filerGroup, clientType string, clientAddress pb.ServerAddress) (clientName string, messageChan chan *master_pb.KeepConnectedResponse) { + clientName = filerGroup + "." + clientType + "@" + string(clientAddress) glog.V(0).Infof("+ client %v", clientName) // we buffer this because otherwise we end up in a potential deadlock where @@ -253,7 +336,7 @@ func (ms *MasterServer) addClient(clientType string, clientAddress string) (clie // trying to send to it in SendHeartbeat and so we can't lock the // clientChansLock to remove the channel and we're stuck writing to it // 100 is probably overkill - messageChan = make(chan *master_pb.VolumeLocation, 100) + messageChan = make(chan *master_pb.KeepConnectedResponse, 100) ms.clientChansLock.Lock() ms.clientChans[clientName] = messageChan @@ -284,25 +367,12 @@ func findClientAddress(ctx context.Context, grpcPort uint32) string { } if tcpAddr, ok := pr.Addr.(*net.TCPAddr); ok { externalIP := tcpAddr.IP - return fmt.Sprintf("%s:%d", externalIP, grpcPort) + return util.JoinHostPort(externalIP.String(), int(grpcPort)) } return pr.Addr.String() } -func (ms *MasterServer) ListMasterClients(ctx context.Context, req *master_pb.ListMasterClientsRequest) (*master_pb.ListMasterClientsResponse, error) { - resp := &master_pb.ListMasterClientsResponse{} - ms.clientChansLock.RLock() - defer ms.clientChansLock.RUnlock() - - for k := range ms.clientChans { - if strings.HasPrefix(k, req.ClientType+"@") { - resp.GrpcAddresses = append(resp.GrpcAddresses, k[len(req.ClientType)+1:]) - } - } - return resp, nil -} - func (ms *MasterServer) GetMasterConfiguration(ctx context.Context, req *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) { // tell the volume servers about the leader @@ -315,7 +385,7 @@ func (ms *MasterServer) GetMasterConfiguration(ctx context.Context, req *master_ DefaultReplication: ms.option.DefaultReplicaPlacement, VolumeSizeLimitMB: uint32(ms.option.VolumeSizeLimitMB), VolumePreallocate: ms.option.VolumePreallocate, - Leader: leader, + Leader: string(leader), } return resp, nil diff --git a/weed/server/master_grpc_server_admin.go b/weed/server/master_grpc_server_admin.go index 93c9e4e4e..1f37e979a 100644 --- a/weed/server/master_grpc_server_admin.go +++ b/weed/server/master_grpc_server_admin.go @@ -3,7 +3,11 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "math/rand" "sync" "time" @@ -62,6 +66,7 @@ type AdminLock struct { accessSecret int64 accessLockTime time.Time lastClient string + lastMessage string } type AdminLocks struct { @@ -75,15 +80,15 @@ func NewAdminLocks() *AdminLocks { } } -func (locks *AdminLocks) isLocked(lockName string) (clientName string, isLocked bool) { +func (locks *AdminLocks) isLocked(lockName string) (clientName string, message string, isLocked bool) { locks.RLock() defer locks.RUnlock() adminLock, found := locks.locks[lockName] if !found { - return "", false + return "", "", false } - glog.V(4).Infof("isLocked %v", adminLock.lastClient) - return adminLock.lastClient, adminLock.accessLockTime.Add(LockDuration).After(time.Now()) + glog.V(4).Infof("isLocked %v: %v", adminLock.lastClient, adminLock.lastMessage) + return adminLock.lastClient, adminLock.lastMessage, adminLock.accessLockTime.Add(LockDuration).After(time.Now()) } func (locks *AdminLocks) isValidToken(lockName string, ts time.Time, token int64) bool { @@ -117,7 +122,7 @@ func (locks *AdminLocks) deleteLock(lockName string) { func (ms *MasterServer) LeaseAdminToken(ctx context.Context, req *master_pb.LeaseAdminTokenRequest) (*master_pb.LeaseAdminTokenResponse, error) { resp := &master_pb.LeaseAdminTokenResponse{} - if lastClient, isLocked := ms.adminLocks.isLocked(req.LockName); isLocked { + if lastClient, lastMessage, isLocked := ms.adminLocks.isLocked(req.LockName); isLocked { glog.V(4).Infof("LeaseAdminToken %v", lastClient) if req.PreviousToken != 0 && ms.adminLocks.isValidToken(req.LockName, time.Unix(0, req.PreviousLockTime), req.PreviousToken) { // for renew @@ -126,7 +131,7 @@ func (ms *MasterServer) LeaseAdminToken(ctx context.Context, req *master_pb.Leas return resp, nil } // refuse since still locked - return resp, fmt.Errorf("already locked by " + lastClient) + return resp, fmt.Errorf("already locked by %v: %v", lastClient, lastMessage) } // for fresh lease request ts, token := ms.adminLocks.generateToken(req.LockName, req.ClientName) @@ -141,3 +146,41 @@ func (ms *MasterServer) ReleaseAdminToken(ctx context.Context, req *master_pb.Re } return resp, nil } + +func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (resp *master_pb.PingResponse, pingErr error) { + resp = &master_pb.PingResponse{ + StartTimeNs: time.Now().UnixNano(), + } + if req.TargetType == cluster.FilerType { + pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.VolumeServerType { + pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.MasterType { + pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client master_pb.SeaweedClient) error { + pingResp, err := client.Ping(ctx, &master_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if pingErr != nil { + pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr) + } + resp.StopTimeNs = time.Now().UnixNano() + return +} diff --git a/weed/server/master_grpc_server_cluster.go b/weed/server/master_grpc_server_cluster.go new file mode 100644 index 000000000..fea4a66aa --- /dev/null +++ b/weed/server/master_grpc_server_cluster.go @@ -0,0 +1,41 @@ +package weed_server + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "math/rand" +) + +func (ms *MasterServer) ListClusterNodes(ctx context.Context, req *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) { + resp := &master_pb.ListClusterNodesResponse{} + filerGroup := cluster.FilerGroup(req.FilerGroup) + clusterNodes := ms.Cluster.ListClusterNode(filerGroup, req.ClientType) + + for _, node := range clusterNodes { + resp.ClusterNodes = append(resp.ClusterNodes, &master_pb.ListClusterNodesResponse_ClusterNode{ + Address: string(node.Address), + Version: node.Version, + IsLeader: ms.Cluster.IsOneLeader(filerGroup, node.Address), + CreatedAtNs: node.CreatedTs.UnixNano(), + }) + } + return resp, nil +} + +func (ms *MasterServer) GetOneFiler(filerGroup cluster.FilerGroup) pb.ServerAddress { + + clusterNodes := ms.Cluster.ListClusterNode(filerGroup, cluster.FilerType) + + var filers []pb.ServerAddress + for _, node := range clusterNodes { + if ms.Cluster.IsOneLeader(filerGroup, node.Address) { + filers = append(filers, node.Address) + } + } + if len(filers) > 0 { + return filers[rand.Intn(len(filers))] + } + return "localhost:8888" +} diff --git a/weed/server/master_grpc_server_collection.go b/weed/server/master_grpc_server_collection.go index b92d6bcbe..654da6b3c 100644 --- a/weed/server/master_grpc_server_collection.go +++ b/weed/server/master_grpc_server_collection.go @@ -58,7 +58,7 @@ func (ms *MasterServer) doDeleteNormalCollection(collectionName string) error { } for _, server := range collection.ListVolumeServers() { - err := operation.WithVolumeServerClient(server.Url(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, server.ServerAddress(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { _, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{ Collection: collectionName, }) @@ -78,7 +78,7 @@ func (ms *MasterServer) doDeleteEcCollection(collectionName string) error { listOfEcServers := ms.Topo.ListEcServersByCollection(collectionName) for _, server := range listOfEcServers { - err := operation.WithVolumeServerClient(server, ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, server, ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { _, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{ Collection: collectionName, }) diff --git a/weed/server/master_grpc_server_raft.go b/weed/server/master_grpc_server_raft.go new file mode 100644 index 000000000..258b6beac --- /dev/null +++ b/weed/server/master_grpc_server_raft.go @@ -0,0 +1,79 @@ +package weed_server + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/hashicorp/raft" +) + +func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_pb.RaftListClusterServersRequest) (*master_pb.RaftListClusterServersResponse, error) { + resp := &master_pb.RaftListClusterServersResponse{} + + if ms.Topo.HashicorpRaft == nil { + return resp, nil + } + + servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers + + for _, server := range servers { + resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{ + Id: string(server.ID), + Address: string(server.Address), + Suffrage: server.Suffrage.String(), + }) + } + return resp, nil +} + +func (ms *MasterServer) RaftAddServer(ctx context.Context, req *master_pb.RaftAddServerRequest) (*master_pb.RaftAddServerResponse, error) { + resp := &master_pb.RaftAddServerResponse{} + + if ms.Topo.HashicorpRaft == nil { + return resp, nil + } + + if ms.Topo.HashicorpRaft.State() != raft.Leader { + return nil, fmt.Errorf("raft add server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String()) + } + + var idxFuture raft.IndexFuture + if req.Voter { + idxFuture = ms.Topo.HashicorpRaft.AddVoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0) + } else { + idxFuture = ms.Topo.HashicorpRaft.AddNonvoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0) + } + + if err := idxFuture.Error(); err != nil { + return nil, err + } + return resp, nil +} + +func (ms *MasterServer) RaftRemoveServer(ctx context.Context, req *master_pb.RaftRemoveServerRequest) (*master_pb.RaftRemoveServerResponse, error) { + resp := &master_pb.RaftRemoveServerResponse{} + + if ms.Topo.HashicorpRaft == nil { + return resp, nil + } + + if ms.Topo.HashicorpRaft.State() != raft.Leader { + return nil, fmt.Errorf("raft remove server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String()) + } + + if !req.Force { + ms.clientChansLock.RLock() + _, ok := ms.clientChans[fmt.Sprintf("%s@%s", cluster.MasterType, req.Id)] + ms.clientChansLock.RUnlock() + if ok { + return resp, fmt.Errorf("raft remove server %s failed: client connection to master exists", req.Id) + } + } + + idxFuture := ms.Topo.HashicorpRaft.RemoveServer(raft.ServerID(req.Id), 0, 0) + if err := idxFuture.Error(); err != nil { + return nil, err + } + return resp, nil +} diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 4b975a0c4..0382c2dae 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -42,15 +42,24 @@ func (ms *MasterServer) ProcessGrowRequest() { return !found }) + option := req.Option + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + // not atomic but it's okay - if !found && ms.shouldVolumeGrow(req.Option) { + if !found && vl.ShouldGrowVolumes(option) { filter.Store(req, nil) // we have lock called inside vg go func() { glog.V(1).Infoln("starting automatic volume grow") start := time.Now() - _, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count) + newVidLocations, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count) glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start)) + if err == nil { + for _, newVidLocation := range newVidLocations { + ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: newVidLocation}) + } + } + vl.DoneGrowRequest() if req.ErrCh != nil { req.ErrCh <- err @@ -82,7 +91,7 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV } var auth string if strings.Contains(result.VolumeOrFileId, ",") { // this is a file id - auth = string(security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, result.VolumeOrFileId)) + auth = string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, result.VolumeOrFileId)) } resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ VolumeOrFileId: result.VolumeOrFileId, @@ -130,10 +139,13 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb, } - if ms.shouldVolumeGrow(option) { + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + + if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) { if ms.Topo.AvailableSpaceFor(option) <= 0 { return nil, fmt.Errorf("no free volumes left for " + option.String()) } + vl.AddGrowRequest() ms.vgCh <- &topology.VolumeGrowRequest{ Option: option, Count: int(req.WritableVolumeCount), @@ -147,14 +159,27 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest ) for time.Now().Sub(startTime) < maxTimeout { - fid, count, dn, err := ms.Topo.PickForWrite(req.Count, option) + fid, count, dnList, err := ms.Topo.PickForWrite(req.Count, option) if err == nil { + dn := dnList.Head() + var replicas []*master_pb.Location + for _, r := range dnList.Rest() { + replicas = append(replicas, &master_pb.Location{ + Url: r.Url(), + PublicUrl: r.PublicUrl, + GrpcPort: uint32(r.GrpcPort), + }) + } return &master_pb.AssignResponse{ - Fid: fid, - Url: dn.Url(), - PublicUrl: dn.PublicUrl, - Count: count, - Auth: string(security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), + Fid: fid, + Location: &master_pb.Location{ + Url: dn.Url(), + PublicUrl: dn.PublicUrl, + GrpcPort: uint32(dn.GrpcPort), + }, + Count: count, + Auth: string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), + Replicas: replicas, }, nil } //glog.V(4).Infoln("waiting for volume growing...") @@ -248,7 +273,7 @@ func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumV resp := &master_pb.VacuumVolumeResponse{} - ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.preallocateSize) + ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), req.VolumeId, req.Collection, ms.preallocateSize) return resp, nil } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index d2edeb6cb..9bf840f08 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -1,7 +1,9 @@ package weed_server import ( + "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/stats" "net/http" "net/http/httputil" "net/url" @@ -11,8 +13,12 @@ import ( "sync" "time" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/raft" "github.com/gorilla/mux" + hashicorpRaft "github.com/hashicorp/raft" "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/glog" @@ -26,14 +32,13 @@ import ( ) const ( - SequencerType = "master.sequencer.type" - SequencerEtcdUrls = "master.sequencer.sequencer_etcd_urls" - SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id" + SequencerType = "master.sequencer.type" + SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id" + RaftServerRemovalTime = 72 * time.Minute ) type MasterOption struct { - Host string - Port int + Master pb.ServerAddress MetaFolder string VolumeSizeLimitMB uint32 VolumePreallocate bool @@ -48,6 +53,7 @@ type MasterOption struct { } type MasterServer struct { + master_pb.UnimplementedSeaweedServer option *MasterOption guard *security.Guard @@ -59,18 +65,23 @@ type MasterServer struct { boundedLeaderChan chan int + onPeerUpdatDoneCn chan string + onPeerUpdatDoneCnExist bool + // notifying clients clientChansLock sync.RWMutex - clientChans map[string]chan *master_pb.VolumeLocation + clientChans map[string]chan *master_pb.KeepConnectedResponse grpcDialOption grpc.DialOption MasterClient *wdclient.MasterClient adminLocks *AdminLocks + + Cluster *cluster.Cluster } -func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *MasterServer { +func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer { v := util.GetViper() signingKey := v.GetString("jwt.signing.key") @@ -100,12 +111,16 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *Maste option: option, preallocateSize: preallocateSize, vgCh: make(chan *topology.VolumeGrowRequest, 1<<6), - clientChans: make(map[string]chan *master_pb.VolumeLocation), + clientChans: make(map[string]chan *master_pb.KeepConnectedResponse), grpcDialOption: grpcDialOption, - MasterClient: wdclient.NewMasterClient(grpcDialOption, "master", option.Host, 0, "", peers), + MasterClient: wdclient.NewMasterClient(grpcDialOption, "", cluster.MasterType, option.Master, "", peers), adminLocks: NewAdminLocks(), + Cluster: cluster.NewCluster(), } ms.boundedLeaderChan = make(chan int, 16) + ms.onPeerUpdatDoneCn = make(chan string) + + ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate seq := ms.createSequencer(option) if nil == seq { @@ -154,18 +169,41 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *Maste } func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) { - ms.Topo.RaftServer = raftServer.raftServer - ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) { - glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value()) - if ms.Topo.RaftServer.Leader() != "" { - glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.") - } - }) + var raftServerName string + if raftServer.raftServer != nil { + ms.Topo.RaftServer = raftServer.raftServer + ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) { + glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value()) + stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc() + if ms.Topo.RaftServer.Leader() != "" { + glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.") + } + }) + raftServerName = ms.Topo.RaftServer.Name() + } else if raftServer.RaftHashicorp != nil { + ms.Topo.HashicorpRaft = raftServer.RaftHashicorp + leaderCh := raftServer.RaftHashicorp.LeaderCh() + prevLeader := ms.Topo.HashicorpRaft.Leader() + go func() { + for { + select { + case isLeader := <-leaderCh: + leader := ms.Topo.HashicorpRaft.Leader() + glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader) + stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc() + prevLeader = leader + } + } + }() + raftServerName = ms.Topo.HashicorpRaft.String() + } if ms.Topo.IsLeader() { - glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!") + glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!") } else { - if ms.Topo.RaftServer.Leader() != "" { + if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" { glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.") + } else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" { + glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.") } } } @@ -174,71 +212,70 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if ms.Topo.IsLeader() { f(w, r) - } else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" { - ms.boundedLeaderChan <- 1 - defer func() { <-ms.boundedLeaderChan }() - targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader()) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, - fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err)) - return - } - glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader()) - proxy := httputil.NewSingleHostReverseProxy(targetUrl) - director := proxy.Director - proxy.Director = func(req *http.Request) { - actualHost, err := security.GetActualRemoteHost(req) - if err == nil { - req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost) - } - director(req) - } - proxy.Transport = util.Transport - proxy.ServeHTTP(w, r) - } else { - // handle requests locally + return + } + var raftServerLeader string + if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" { + raftServerLeader = ms.Topo.RaftServer.Leader() + } else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" { + raftServerLeader = string(ms.Topo.HashicorpRaft.Leader()) + } + if raftServerLeader == "" { f(w, r) + return + } + ms.boundedLeaderChan <- 1 + defer func() { <-ms.boundedLeaderChan }() + targetUrl, err := url.Parse("http://" + raftServerLeader) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, + fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err)) + return } + glog.V(4).Infoln("proxying to leader", raftServerLeader) + proxy := httputil.NewSingleHostReverseProxy(targetUrl) + director := proxy.Director + proxy.Director = func(req *http.Request) { + actualHost, err := security.GetActualRemoteHost(req) + if err == nil { + req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost) + } + director(req) + } + proxy.Transport = util.Transport + proxy.ServeHTTP(w, r) } } func (ms *MasterServer) startAdminScripts() { - var err error v := util.GetViper() adminScripts := v.GetString("master.maintenance.scripts") - glog.V(0).Infof("adminScripts:\n%v", adminScripts) if adminScripts == "" { return } + glog.V(0).Infof("adminScripts: %v", adminScripts) v.SetDefault("master.maintenance.sleep_minutes", 17) sleepMinutes := v.GetInt("master.maintenance.sleep_minutes") - v.SetDefault("master.filer.default", "localhost:8888") - filerHostPort := v.GetString("master.filer.default") - scriptLines := strings.Split(adminScripts, "\n") if !strings.Contains(adminScripts, "lock") { scriptLines = append(append([]string{}, "lock"), scriptLines...) scriptLines = append(scriptLines, "unlock") } - masterAddress := fmt.Sprintf("%s:%d", ms.option.Host, ms.option.Port) + masterAddress := string(ms.option.Master) var shellOptions shell.ShellOptions shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master") shellOptions.Masters = &masterAddress - shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(filerHostPort) - shellOptions.FilerAddress = filerHostPort shellOptions.Directory = "/" - if err != nil { - glog.V(0).Infof("failed to parse master.filer.default = %s : %v\n", filerHostPort, err) - return - } + emptyFilerGroup := "" + shellOptions.FilerGroup = &emptyFilerGroup - commandEnv := shell.NewCommandEnv(shellOptions) + commandEnv := shell.NewCommandEnv(&shellOptions) reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) @@ -247,9 +284,13 @@ func (ms *MasterServer) startAdminScripts() { go func() { commandEnv.MasterClient.WaitUntilConnected() - c := time.Tick(time.Duration(sleepMinutes) * time.Minute) - for range c { + for { + time.Sleep(time.Duration(sleepMinutes) * time.Minute) if ms.Topo.IsLeader() { + shellOptions.FilerAddress = ms.GetOneFiler(cluster.FilerGroup(*shellOptions.FilerGroup)) + if shellOptions.FilerAddress == "" { + continue + } for _, line := range scriptLines { for _, c := range strings.Split(line, ";") { processEachCmd(reg, c, commandEnv) @@ -287,19 +328,10 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer seqType := strings.ToLower(v.GetString(SequencerType)) glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType) switch strings.ToLower(seqType) { - case "etcd": - var err error - urls := v.GetString(SequencerEtcdUrls) - glog.V(0).Infof("[%s] : [%s]", SequencerEtcdUrls, urls) - seq, err = sequence.NewEtcdSequencer(urls, option.MetaFolder) - if err != nil { - glog.Error(err) - seq = nil - } case "snowflake": var err error snowflakeId := v.GetInt(SequencerSnowflakeId) - seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port), snowflakeId) + seq, err = sequence.NewSnowflakeSequencer(string(option.Master), snowflakeId) if err != nil { glog.Error(err) seq = nil @@ -309,3 +341,57 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer } return seq } + +func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) { + if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil { + return + } + glog.V(4).Infof("OnPeerUpdate: %+v", update) + + peerAddress := pb.ServerAddress(update.Address) + peerName := string(peerAddress) + isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader + if update.IsAdd { + if isLeader { + raftServerFound := false + for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers { + if string(server.ID) == peerName { + raftServerFound = true + } + } + if !raftServerFound { + glog.V(0).Infof("adding new raft server: %s", peerName) + ms.Topo.HashicorpRaft.AddVoter( + hashicorpRaft.ServerID(peerName), + hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0) + } + } + if ms.onPeerUpdatDoneCnExist { + ms.onPeerUpdatDoneCn <- peerName + } + } else if isLeader { + go func(peerName string) { + for { + select { + case <-time.After(RaftServerRemovalTime): + err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + _, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{ + Id: peerName, + Force: false, + }) + return err + }) + if err != nil { + glog.Warningf("failed to removing old raft server %s: %v", peerName, err) + } + return + case peerDone := <-ms.onPeerUpdatDoneCn: + if peerName == peerDone { + return + } + } + } + }(peerName) + ms.onPeerUpdatDoneCnExist = true + } +} diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 2a1f6d523..0b79c4ed5 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -113,13 +113,16 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if ms.shouldVolumeGrow(option) { + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + + if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) { glog.V(0).Infof("dirAssign volume growth %v from %v", option.String(), r.RemoteAddr) if ms.Topo.AvailableSpaceFor(option) <= 0 { writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left for " + option.String()}) return } errCh := make(chan error, 1) + vl.AddGrowRequest() ms.vgCh <- &topology.VolumeGrowRequest{ Option: option, Count: writableVolumeCount, @@ -130,9 +133,10 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } } - fid, count, dn, err := ms.Topo.PickForWrite(requestedCount, option) + fid, count, dnList, err := ms.Topo.PickForWrite(requestedCount, option) if err == nil { ms.maybeAddJwtAuthorization(w, fid, true) + dn := dnList.Head() writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) } else { writeJsonQuiet(w, r, http.StatusNotAcceptable, operation.AssignResult{Error: err.Error()}) @@ -145,9 +149,9 @@ func (ms *MasterServer) maybeAddJwtAuthorization(w http.ResponseWriter, fileId s } var encodedJwt security.EncodedJwt if isWrite { - encodedJwt = security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fileId) + encodedJwt = security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fileId) } else { - encodedJwt = security.GenJwt(ms.guard.ReadSigningKey, ms.guard.ReadExpiresAfterSec, fileId) + encodedJwt = security.GenJwtForVolumeServer(ms.guard.ReadSigningKey, ms.guard.ReadExpiresAfterSec, fileId) } if encodedJwt == "" { return diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index 4a86348d9..47abfb892 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -3,6 +3,8 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "math/rand" "net/http" "strconv" @@ -26,7 +28,7 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R return } for _, server := range collection.ListVolumeServers() { - err := operation.WithVolumeServerClient(server.Url(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, server.ServerAddress(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { _, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{ Collection: collection.Name, }) @@ -63,7 +65,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque } } // glog.Infoln("garbageThreshold =", gcThreshold) - ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize) + ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, 0, "", ms.preallocateSize) ms.dirStatusHandler(w, r) } @@ -80,7 +82,9 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request if ms.Topo.AvailableSpaceFor(option) < int64(count*option.ReplicaPlacement.GetCopyCount()) { err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.AvailableSpaceFor(option), count*option.ReplicaPlacement.GetCopyCount()) } else { - count, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, count, option, ms.Topo) + var newVidLocations []*master_pb.VolumeLocation + newVidLocations, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, count, option, ms.Topo) + count = len(newVidLocations) } } else { err = fmt.Errorf("can not parse parameter count %s", r.FormValue("count")) @@ -118,32 +122,19 @@ func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) } } -func (ms *MasterServer) selfUrl(r *http.Request) string { - if r.Host != "" { - return r.Host - } - return "localhost:" + strconv.Itoa(ms.option.Port) -} func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) { if ms.Topo.IsLeader() { - submitForClientHandler(w, r, func() string { return ms.selfUrl(r) }, ms.grpcDialOption) + submitForClientHandler(w, r, func() pb.ServerAddress { return ms.option.Master }, ms.grpcDialOption) } else { masterUrl, err := ms.Topo.Leader() if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) } else { - submitForClientHandler(w, r, func() string { return masterUrl }, ms.grpcDialOption) + submitForClientHandler(w, r, func() pb.ServerAddress { return masterUrl }, ms.grpcDialOption) } } } -func (ms *MasterServer) shouldVolumeGrow(option *topology.VolumeGrowOption) bool { - vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) - active, high := vl.GetActiveVolumeCount(option) - //glog.V(0).Infof("active volume: %d, high usage volume: %d\n", active, high) - return active <= high -} - func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) { replicationString := r.FormValue("replication") if replicationString == "" { diff --git a/weed/server/master_server_handlers_ui.go b/weed/server/master_server_handlers_ui.go index 015bfbd00..d8260d8d2 100644 --- a/weed/server/master_server_handlers_ui.go +++ b/weed/server/master_server_handlers_ui.go @@ -5,6 +5,8 @@ import ( "time" "github.com/chrislusf/raft" + hashicorpRaft "github.com/hashicorp/raft" + ui "github.com/chrislusf/seaweedfs/weed/server/master_ui" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" @@ -13,20 +15,40 @@ import ( func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { infos := make(map[string]interface{}) infos["Up Time"] = time.Now().Sub(startTime).String() - args := struct { - Version string - Topology interface{} - RaftServer raft.Server - Stats map[string]interface{} - Counters *stats.ServerStats - VolumeSizeLimitMB uint32 - }{ - util.Version(), - ms.Topo.ToMap(), - ms.Topo.RaftServer, - infos, - serverStats, - ms.option.VolumeSizeLimitMB, + infos["Max Volume Id"] = ms.Topo.GetMaxVolumeId() + if ms.Topo.RaftServer != nil { + args := struct { + Version string + Topology interface{} + RaftServer raft.Server + Stats map[string]interface{} + Counters *stats.ServerStats + VolumeSizeLimitMB uint32 + }{ + util.Version(), + ms.Topo.ToMap(), + ms.Topo.RaftServer, + infos, + serverStats, + ms.option.VolumeSizeLimitMB, + } + ui.StatusTpl.Execute(w, args) + } else if ms.Topo.HashicorpRaft != nil { + args := struct { + Version string + Topology interface{} + RaftServer *hashicorpRaft.Raft + Stats map[string]interface{} + Counters *stats.ServerStats + VolumeSizeLimitMB uint32 + }{ + util.Version(), + ms.Topo.ToMap(), + ms.Topo.HashicorpRaft, + infos, + serverStats, + ms.option.VolumeSizeLimitMB, + } + ui.StatusNewRaftTpl.Execute(w, args) } - ui.StatusTpl.Execute(w, args) } diff --git a/weed/server/master_ui/masterNewRaft.html b/weed/server/master_ui/masterNewRaft.html new file mode 100644 index 000000000..32afdceac --- /dev/null +++ b/weed/server/master_ui/masterNewRaft.html @@ -0,0 +1,121 @@ + + + + SeaweedFS {{ .Version }} + + + +
+
+

+ + SeaweedFS {{ .Version }} +

+
+ +
+
+

Cluster status

+ + + + + + + + + + + + + + + {{ with .RaftServer }} + + + + + + + + + {{ end }} + +
Volume Size Limit{{ .VolumeSizeLimitMB }}MB
Free{{ .Topology.Free }}
Max{{ .Topology.Max }}
Leader{{ .Leader }}
Other Masters +
    + {{ range $k, $p := .GetConfiguration.Configuration.Servers }} +
  • {{ $p.ID }}
    • + {{ end }} +
+
+
+ +
+

System Stats

+ + + + + + {{ range $key, $val := .Stats }} + + + + + {{ end }} +
Concurrent Connections{{ .Counters.Connections.WeekCounter.Sum }}
{{ $key }}{{ $val }}
+

Raft Stats

+ + + + + + + + + +
applied_index{{ .RaftServer.Stats.applied_index }}
last_log_term{{ .RaftServer.Stats.last_log_term }}
+
+
+ +
+

Topology

+ + + + + + + + + + + + + + {{ range $dc_index, $dc := .Topology.DataCenters }} + {{ range $rack_index, $rack := $dc.Racks }} + {{ range $dn_index, $dn := $rack.DataNodes }} + + + + + + + + + + {{ end }} + {{ end }} + {{ end }} + +
Data CenterRackRemoteAddr#VolumesVolume Ids#ErasureCodingShardsMax
{{ $dc.Id }}{{ $rack.Id }}{{ $dn.Url }} + {{ if ne $dn.PublicUrl $dn.Url }} + / {{ $dn.PublicUrl }} + {{ end }} + {{ $dn.Volumes }}{{ $dn.VolumeIds}}{{ $dn.EcShards }}{{ $dn.Max }}
+
+ +
+ + diff --git a/weed/server/master_ui/templates.go b/weed/server/master_ui/templates.go index 415022b97..a6dcc57d7 100644 --- a/weed/server/master_ui/templates.go +++ b/weed/server/master_ui/templates.go @@ -8,4 +8,8 @@ import ( //go:embed master.html var masterHtml string +//go:embed masterNewRaft.html +var masterNewRaftHtml string + var StatusTpl = template.Must(template.New("status").Parse(masterHtml)) +var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml)) diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go new file mode 100644 index 000000000..9971eaa48 --- /dev/null +++ b/weed/server/raft_hashicorp.go @@ -0,0 +1,186 @@ +package weed_server + +// https://yusufs.medium.com/creating-distributed-kv-database-by-implementing-raft-consensus-using-golang-d0884eef2e28 +// https://github.com/Jille/raft-grpc-example/blob/cd5bcab0218f008e044fbeee4facdd01b06018ad/application.go#L18 + +import ( + "fmt" + transport "github.com/Jille/raft-grpc-transport" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/hashicorp/raft" + boltdb "github.com/hashicorp/raft-boltdb" + "google.golang.org/grpc" + "math/rand" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + ldbFile = "logs.dat" + sdbFile = "stable.dat" + updatePeersTimeout = 15 * time.Minute +) + +func getPeerIdx(self pb.ServerAddress, mapPeers map[string]pb.ServerAddress) int { + peers := make([]pb.ServerAddress, 0, len(mapPeers)) + for _, peer := range mapPeers { + peers = append(peers, peer) + } + sort.Slice(peers, func(i, j int) bool { + return strings.Compare(string(peers[i]), string(peers[j])) < 0 + }) + for i, peer := range peers { + if string(peer) == string(self) { + return i + } + } + return -1 +} + +func (s *RaftServer) AddPeersConfiguration() (cfg raft.Configuration) { + for _, peer := range s.peers { + cfg.Servers = append(cfg.Servers, raft.Server{ + Suffrage: raft.Voter, + ID: raft.ServerID(peer), + Address: raft.ServerAddress(peer.ToGrpcAddress()), + }) + } + return cfg +} + +func (s *RaftServer) UpdatePeers() { + for { + select { + case isLeader := <-s.RaftHashicorp.LeaderCh(): + if isLeader { + peerLeader := string(s.serverAddr) + existsPeerName := make(map[string]bool) + for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers { + if string(server.ID) == peerLeader { + continue + } + existsPeerName[string(server.ID)] = true + } + for _, peer := range s.peers { + peerName := string(peer) + if peerName == peerLeader || existsPeerName[peerName] { + continue + } + glog.V(0).Infof("adding new peer: %s", peerName) + s.RaftHashicorp.AddVoter( + raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0) + } + for peer, _ := range existsPeerName { + if _, found := s.peers[peer]; !found { + glog.V(0).Infof("removing old peer: %s", peer) + s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0) + } + } + if _, found := s.peers[peerLeader]; !found { + glog.V(0).Infof("removing old leader peer: %s", peerLeader) + s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0) + } + } + return + case <-time.After(updatePeersTimeout): + return + } + } +} + +func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { + s := &RaftServer{ + peers: option.Peers, + serverAddr: option.ServerAddr, + dataDir: option.DataDir, + topo: option.Topo, + } + + c := raft.DefaultConfig() + c.LocalID = raft.ServerID(s.serverAddr) // TODO maybee the IP:port address will change + c.HeartbeatTimeout = time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1)) + c.ElectionTimeout = option.ElectionTimeout + if c.LeaderLeaseTimeout > c.HeartbeatTimeout { + c.LeaderLeaseTimeout = c.HeartbeatTimeout + } + if glog.V(4) { + c.LogLevel = "Debug" + } else if glog.V(2) { + c.LogLevel = "Info" + } else if glog.V(1) { + c.LogLevel = "Warn" + } else if glog.V(0) { + c.LogLevel = "Error" + } + + if option.RaftBootstrap { + os.RemoveAll(path.Join(s.dataDir, ldbFile)) + os.RemoveAll(path.Join(s.dataDir, sdbFile)) + os.RemoveAll(path.Join(s.dataDir, "snapshots")) + } + if err := os.MkdirAll(path.Join(s.dataDir, "snapshots"), os.ModePerm); err != nil { + return nil, err + } + baseDir := s.dataDir + + ldb, err := boltdb.NewBoltStore(filepath.Join(baseDir, ldbFile)) + if err != nil { + return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "logs.dat"), err) + } + + sdb, err := boltdb.NewBoltStore(filepath.Join(baseDir, sdbFile)) + if err != nil { + return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "stable.dat"), err) + } + + fss, err := raft.NewFileSnapshotStore(baseDir, 3, os.Stderr) + if err != nil { + return nil, fmt.Errorf(`raft.NewFileSnapshotStore(%q, ...): %v`, baseDir, err) + } + + s.TransportManager = transport.New(raft.ServerAddress(s.serverAddr), []grpc.DialOption{option.GrpcDialOption}) + + stateMachine := StateMachine{topo: option.Topo} + s.RaftHashicorp, err = raft.NewRaft(c, &stateMachine, ldb, sdb, fss, s.TransportManager.Transport()) + if err != nil { + return nil, fmt.Errorf("raft.NewRaft: %v", err) + } + if option.RaftBootstrap || len(s.RaftHashicorp.GetConfiguration().Configuration().Servers) == 0 { + cfg := s.AddPeersConfiguration() + // Need to get lock, in case all servers do this at the same time. + peerIdx := getPeerIdx(s.serverAddr, s.peers) + timeSpeep := time.Duration(float64(c.LeaderLeaseTimeout) * (rand.Float64()*0.25 + 1) * float64(peerIdx)) + glog.V(0).Infof("Bootstrapping idx: %d sleep: %v new cluster: %+v", peerIdx, timeSpeep, cfg) + time.Sleep(timeSpeep) + f := s.RaftHashicorp.BootstrapCluster(cfg) + if err := f.Error(); err != nil { + return nil, fmt.Errorf("raft.Raft.BootstrapCluster: %v", err) + } + } else { + go s.UpdatePeers() + } + + ticker := time.NewTicker(c.HeartbeatTimeout * 10) + if glog.V(4) { + go func() { + for { + select { + case <-ticker.C: + cfuture := s.RaftHashicorp.GetConfiguration() + if err = cfuture.Error(); err != nil { + glog.Fatalf("error getting config: %s", err) + } + configuration := cfuture.Configuration() + glog.V(4).Infof("Showing peers known by %s:\n%+v", s.RaftHashicorp.String(), configuration.Servers) + } + } + }() + } + + return s, nil +} diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index 85841e409..ad0a1c8ce 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -2,10 +2,12 @@ package weed_server import ( "encoding/json" + transport "github.com/Jille/raft-grpc-transport" + "io" + "io/ioutil" "math/rand" "os" "path" - "sort" "time" "google.golang.org/grpc" @@ -13,17 +15,32 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/raft" + hashicorpRaft "github.com/hashicorp/raft" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/topology" ) +type RaftServerOption struct { + GrpcDialOption grpc.DialOption + Peers map[string]pb.ServerAddress + ServerAddr pb.ServerAddress + DataDir string + Topo *topology.Topology + RaftResumeState bool + HeartbeatInterval time.Duration + ElectionTimeout time.Duration + RaftBootstrap bool +} + type RaftServer struct { - peers []string // initial peers to join with - raftServer raft.Server - dataDir string - serverAddr string - topo *topology.Topology + peers map[string]pb.ServerAddress // initial peers to join with + raftServer raft.Server + RaftHashicorp *hashicorpRaft.Raft + TransportManager *transport.Manager + dataDir string + serverAddr pb.ServerAddress + topo *topology.Topology *raft.GrpcServer } @@ -32,6 +49,8 @@ type StateMachine struct { topo *topology.Topology } +var _ hashicorpRaft.FSM = &StateMachine{} + func (s StateMachine) Save() ([]byte, error) { state := topology.MaxVolumeIdCommand{ MaxVolumeId: s.topo.GetMaxVolumeId(), @@ -51,12 +70,42 @@ func (s StateMachine) Recovery(data []byte) error { return nil } -func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr, dataDir string, topo *topology.Topology, raftResumeState bool) (*RaftServer, error) { +func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} { + before := s.topo.GetMaxVolumeId() + state := topology.MaxVolumeIdCommand{} + err := json.Unmarshal(l.Data, &state) + if err != nil { + return err + } + s.topo.UpAdjustMaxVolumeId(state.MaxVolumeId) + + glog.V(1).Infoln("max volume id", before, "==>", s.topo.GetMaxVolumeId()) + return nil +} + +func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) { + return &topology.MaxVolumeIdCommand{ + MaxVolumeId: s.topo.GetMaxVolumeId(), + }, nil +} + +func (s *StateMachine) Restore(r io.ReadCloser) error { + b, err := ioutil.ReadAll(r) + if err != nil { + return err + } + if err := s.Recovery(b); err != nil { + return err + } + return nil +} + +func NewRaftServer(option *RaftServerOption) (*RaftServer, error) { s := &RaftServer{ - peers: peers, - serverAddr: serverAddr, - dataDir: dataDir, - topo: topo, + peers: option.Peers, + serverAddr: option.ServerAddr, + dataDir: option.DataDir, + topo: option.Topo, } if glog.V(4) { @@ -66,27 +115,29 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr, d raft.RegisterCommand(&topology.MaxVolumeIdCommand{}) var err error - transporter := raft.NewGrpcTransporter(grpcDialOption) - glog.V(0).Infof("Starting RaftServer with %v", serverAddr) + transporter := raft.NewGrpcTransporter(option.GrpcDialOption) + glog.V(0).Infof("Starting RaftServer with %v", option.ServerAddr) - if !raftResumeState { + // always clear previous log to avoid server is promotable + os.RemoveAll(path.Join(s.dataDir, "log")) + if !option.RaftResumeState { // always clear previous metadata os.RemoveAll(path.Join(s.dataDir, "conf")) - os.RemoveAll(path.Join(s.dataDir, "log")) os.RemoveAll(path.Join(s.dataDir, "snapshot")) } - if err := os.MkdirAll(path.Join(s.dataDir, "snapshot"), 0600); err != nil { + if err := os.MkdirAll(path.Join(s.dataDir, "snapshot"), os.ModePerm); err != nil { return nil, err } - stateMachine := StateMachine{topo: topo} - s.raftServer, err = raft.NewServer(s.serverAddr, s.dataDir, transporter, stateMachine, topo, "") + stateMachine := StateMachine{topo: option.Topo} + s.raftServer, err = raft.NewServer(string(s.serverAddr), s.dataDir, transporter, stateMachine, option.Topo, "") if err != nil { glog.V(0).Infoln(err) return nil, err } - s.raftServer.SetHeartbeatInterval(time.Duration(300+rand.Intn(150)) * time.Millisecond) - s.raftServer.SetElectionTimeout(10 * time.Second) + heartbeatInterval := time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1)) + s.raftServer.SetHeartbeatInterval(heartbeatInterval) + s.raftServer.SetElectionTimeout(option.ElectionTimeout) if err := s.raftServer.LoadSnapshot(); err != nil { return nil, err } @@ -94,68 +145,53 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr, d return nil, err } - for _, peer := range s.peers { - if err := s.raftServer.AddPeer(peer, pb.ServerToGrpcAddress(peer)); err != nil { + for name, peer := range s.peers { + if err := s.raftServer.AddPeer(name, peer.ToGrpcAddress()); err != nil { return nil, err } } // Remove deleted peers for existsPeerName := range s.raftServer.Peers() { - exists, existingPeer := false, "" - for _, peer := range s.peers { - if pb.ServerToGrpcAddress(peer) == existsPeerName { - exists, existingPeer = true, peer - break - } - } - if exists { + if existingPeer, found := s.peers[existsPeerName]; !found { if err := s.raftServer.RemovePeer(existsPeerName); err != nil { glog.V(0).Infoln(err) return nil, err } else { - glog.V(0).Infof("removing old peer %s", existingPeer) + glog.V(0).Infof("removing old peer: %s", existingPeer) } } } s.GrpcServer = raft.NewGrpcServer(s.raftServer) - if s.raftServer.IsLogEmpty() && isTheFirstOne(serverAddr, s.peers) { - // Initialize the server by joining itself. - // s.DoJoinCommand() - } - glog.V(0).Infof("current cluster leader: %v", s.raftServer.Leader()) return s, nil } func (s *RaftServer) Peers() (members []string) { - peers := s.raftServer.Peers() - - for _, p := range peers { - members = append(members, p.Name) + if s.raftServer != nil { + peers := s.raftServer.Peers() + for _, p := range peers { + members = append(members, p.Name) + } + } else if s.RaftHashicorp != nil { + cfg := s.RaftHashicorp.GetConfiguration() + for _, p := range cfg.Configuration().Servers { + members = append(members, string(p.ID)) + } } - return } -func isTheFirstOne(self string, peers []string) bool { - sort.Strings(peers) - if len(peers) <= 0 { - return true - } - return self == peers[0] -} - func (s *RaftServer) DoJoinCommand() { glog.V(0).Infoln("Initializing new cluster") if _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ Name: s.raftServer.Name(), - ConnectionString: pb.ServerToGrpcAddress(s.serverAddr), + ConnectionString: s.serverAddr.ToGrpcAddress(), }); err != nil { glog.Errorf("fail to send join command: %v", err) } diff --git a/weed/server/raft_server_handlers.go b/weed/server/raft_server_handlers.go index 252570eab..cc3e6e37f 100644 --- a/weed/server/raft_server_handlers.go +++ b/weed/server/raft_server_handlers.go @@ -1,15 +1,16 @@ package weed_server import ( + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "net/http" ) type ClusterStatusResult struct { - IsLeader bool `json:"IsLeader,omitempty"` - Leader string `json:"Leader,omitempty"` - Peers []string `json:"Peers,omitempty"` - MaxVolumeId needle.VolumeId `json:"MaxVolumeId,omitempty"` + IsLeader bool `json:"IsLeader,omitempty"` + Leader pb.ServerAddress `json:"Leader,omitempty"` + Peers []string `json:"Peers,omitempty"` + MaxVolumeId needle.VolumeId `json:"MaxVolumeId,omitempty"` } func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request) { @@ -24,3 +25,11 @@ func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request) { } writeJsonQuiet(w, r, http.StatusOK, ret) } + +func (s *RaftServer) StatsRaftHandler(w http.ResponseWriter, r *http.Request) { + if s.RaftHashicorp == nil { + writeJsonQuiet(w, r, http.StatusNotFound, nil) + return + } + writeJsonQuiet(w, r, http.StatusOK, s.RaftHashicorp.Stats()) +} diff --git a/weed/server/volume_grpc_admin.go b/weed/server/volume_grpc_admin.go index 898c3da12..2ffdf2226 100644 --- a/weed/server/volume_grpc_admin.go +++ b/weed/server/volume_grpc_admin.go @@ -3,7 +3,13 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/util" "path/filepath" + "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" @@ -183,7 +189,12 @@ func (vs *VolumeServer) VolumeStatus(ctx context.Context, req *volume_server_pb. func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_server_pb.VolumeServerStatusRequest) (*volume_server_pb.VolumeServerStatusResponse, error) { - resp := &volume_server_pb.VolumeServerStatusResponse{} + resp := &volume_server_pb.VolumeServerStatusResponse{ + MemoryStatus: stats.MemStat(), + Version: util.Version(), + DataCenter: vs.dataCenter, + Rack: vs.rack, + } for _, loc := range vs.store.Locations { if dir, e := filepath.Abs(loc.Directory); e == nil { @@ -191,8 +202,6 @@ func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_serv } } - resp.MemoryStatus = stats.MemStat() - return resp, nil } @@ -247,3 +256,41 @@ func (vs *VolumeServer) VolumeNeedleStatus(ctx context.Context, req *volume_serv return resp, nil } + +func (vs *VolumeServer) Ping(ctx context.Context, req *volume_server_pb.PingRequest) (resp *volume_server_pb.PingResponse, pingErr error) { + resp = &volume_server_pb.PingResponse{ + StartTimeNs: time.Now().UnixNano(), + } + if req.TargetType == cluster.FilerType { + pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.VolumeServerType { + pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if req.TargetType == cluster.MasterType { + pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client master_pb.SeaweedClient) error { + pingResp, err := client.Ping(ctx, &master_pb.PingRequest{}) + if pingResp != nil { + resp.RemoteTimeNs = pingResp.StartTimeNs + } + return err + }) + } + if pingErr != nil { + pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr) + } + resp.StopTimeNs = time.Now().UnixNano() + return +} diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go index f8875169f..078b78eb2 100644 --- a/weed/server/volume_grpc_client_to_master.go +++ b/weed/server/volume_grpc_client_to_master.go @@ -2,9 +2,11 @@ package weed_server import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/operation" + "os" "time" + "github.com/chrislusf/seaweedfs/weed/operation" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb" @@ -19,15 +21,14 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func (vs *VolumeServer) GetMaster() string { +func (vs *VolumeServer) GetMaster() pb.ServerAddress { return vs.currentMaster } func (vs *VolumeServer) checkWithMaster() (err error) { - isConnected := false - for !isConnected { + for { for _, master := range vs.SeedMasterNodes { - err = operation.WithMasterServerClient(master, vs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + err = operation.WithMasterServerClient(false, master, vs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) if err != nil { return fmt.Errorf("get master %s configuration: %v", master, err) @@ -44,7 +45,6 @@ func (vs *VolumeServer) checkWithMaster() (err error) { } time.Sleep(1790 * time.Millisecond) } - return } func (vs *VolumeServer) heartbeat() { @@ -56,7 +56,7 @@ func (vs *VolumeServer) heartbeat() { grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.volume") var err error - var newLeader string + var newLeader pb.ServerAddress for vs.isHeartbeating { for _, master := range vs.SeedMasterNodes { if newLeader != "" { @@ -65,13 +65,8 @@ func (vs *VolumeServer) heartbeat() { time.Sleep(3 * time.Second) master = newLeader } - masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(master) - if parseErr != nil { - glog.V(0).Infof("failed to parse master grpc %v: %v", masterGrpcAddress, parseErr) - continue - } vs.store.MasterAddress = master - newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, grpcDialOption, time.Duration(vs.pulseSeconds)*time.Second) + newLeader, err = vs.doHeartbeat(master, grpcDialOption, time.Duration(vs.pulseSeconds)*time.Second) if err != nil { glog.V(0).Infof("heartbeat error: %v", err) time.Sleep(time.Duration(vs.pulseSeconds) * time.Second) @@ -94,25 +89,25 @@ func (vs *VolumeServer) StopHeartbeat() (isAlreadyStopping bool) { return false } -func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDialOption grpc.DialOption, sleepInterval time.Duration) (newLeader string, err error) { +func (vs *VolumeServer) doHeartbeat(masterAddress pb.ServerAddress, grpcDialOption grpc.DialOption, sleepInterval time.Duration) (newLeader pb.ServerAddress, err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - grpcConection, err := pb.GrpcDial(ctx, masterGrpcAddress, grpcDialOption) + grpcConection, err := pb.GrpcDial(ctx, masterAddress.ToGrpcAddress(), grpcDialOption) if err != nil { - return "", fmt.Errorf("fail to dial %s : %v", masterNode, err) + return "", fmt.Errorf("fail to dial %s : %v", masterAddress, err) } defer grpcConection.Close() client := master_pb.NewSeaweedClient(grpcConection) stream, err := client.SendHeartbeat(ctx) if err != nil { - glog.V(0).Infof("SendHeartbeat to %s: %v", masterNode, err) + glog.V(0).Infof("SendHeartbeat to %s: %v", masterAddress, err) return "", err } - glog.V(0).Infof("Heartbeat to: %v", masterNode) - vs.currentMaster = masterNode + glog.V(0).Infof("Heartbeat to: %v", masterAddress) + vs.currentMaster = masterAddress doneChan := make(chan error, 1) @@ -123,17 +118,30 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi doneChan <- err return } + if len(in.DuplicatedUuids) > 0 { + var duplicateDir []string + for _, loc := range vs.store.Locations { + for _, uuid := range in.DuplicatedUuids { + if uuid == loc.DirectoryUuid { + duplicateDir = append(duplicateDir, loc.Directory) + } + } + } + glog.Errorf("Shut down Volume Server due to duplicate volume directories: %v", duplicateDir) + os.Exit(1) + } if in.GetVolumeSizeLimit() != 0 && vs.store.GetVolumeSizeLimit() != in.GetVolumeSizeLimit() { vs.store.SetVolumeSizeLimit(in.GetVolumeSizeLimit()) if vs.store.MaybeAdjustVolumeMax() { if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", vs.currentMaster, err) + return } } } - if in.GetLeader() != "" && vs.currentMaster != in.GetLeader() { + if in.GetLeader() != "" && string(vs.currentMaster) != in.GetLeader() { glog.V(0).Infof("Volume Server found a new master newLeader: %v instead of %v", in.GetLeader(), vs.currentMaster) - newLeader = in.GetLeader() + newLeader = pb.ServerAddress(in.GetLeader()) doneChan <- nil return } @@ -141,12 +149,12 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi }() if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { - glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterAddress, err) return "", err } if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil { - glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterAddress, err) return "", err } @@ -161,9 +169,9 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi &volumeMessage, }, } - glog.V(1).Infof("volume server %s:%d adds volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id) + glog.V(0).Infof("volume server %s:%d adds volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id) if err = stream.Send(deltaBeat); err != nil { - glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterAddress, err) return "", err } case ecShardMessage := <-vs.store.NewEcShardsChan: @@ -172,10 +180,10 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi &ecShardMessage, }, } - glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, + glog.V(0).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds()) if err = stream.Send(deltaBeat); err != nil { - glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterAddress, err) return "", err } case volumeMessage := <-vs.store.DeletedVolumesChan: @@ -184,9 +192,9 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi &volumeMessage, }, } - glog.V(1).Infof("volume server %s:%d deletes volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id) + glog.V(0).Infof("volume server %s:%d deletes volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id) if err = stream.Send(deltaBeat); err != nil { - glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterAddress, err) return "", err } case ecShardMessage := <-vs.store.DeletedEcShardsChan: @@ -195,23 +203,23 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi &ecShardMessage, }, } - glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, + glog.V(0).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds()) if err = stream.Send(deltaBeat); err != nil { - glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterAddress, err) return "", err } case <-volumeTickChan: glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port) vs.store.MaybeAdjustVolumeMax() if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { - glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterAddress, err) return "", err } case <-ecShardTickChan: glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port) if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil { - glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterAddress, err) return "", err } case err = <-doneChan: @@ -230,7 +238,7 @@ func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDi } glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port) if err = stream.Send(emptyBeat); err != nil { - glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterAddress, err) return "", err } return diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index 2ad77a7ff..b4bc850e2 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -3,26 +3,28 @@ package weed_server import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage/backend" "io" - "io/ioutil" "math" "os" "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" ) const BufferSizeLimit = 1024 * 1024 * 2 // VolumeCopy copy the .idx .dat .vif files, and mount the volume -func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) { +func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stream volume_server_pb.VolumeServer_VolumeCopyServer) error { v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) if v != nil { @@ -31,7 +33,7 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId)) if err != nil { - return nil, fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err) + return fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err) } glog.V(0).Infof("deleted existing volume %d before copying.", req.VolumeId) @@ -45,7 +47,7 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo // confirm size and timestamp var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse var dataBaseFileName, indexBaseFileName, idxFileName, datFileName string - err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(true, pb.ServerAddress(req.SourceDataNode), vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { var err error volFileInfoResp, err = client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{ @@ -67,7 +69,7 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo dataBaseFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId)) indexBaseFileName = storage.VolumeFileName(location.IdxDirectory, volFileInfoResp.Collection, int(req.VolumeId)) - ioutil.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755) + util.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755) defer func() { if err != nil { @@ -78,18 +80,66 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo } }() + var preallocateSize int64 + if grpcErr := pb.WithMasterClient(false, vs.GetMaster(), vs.grpcDialOption, func(client master_pb.SeaweedClient) error { + resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get master %s configuration: %v", vs.GetMaster(), err) + } + if resp.VolumePreallocate { + preallocateSize = int64(resp.VolumeSizeLimitMB) * (1 << 20) + } + return nil + }); grpcErr != nil { + glog.V(0).Infof("connect to %s: %v", vs.GetMaster(), grpcErr) + } + + if preallocateSize > 0 { + volumeFile := dataBaseFileName + ".dat" + _, err := backend.CreateVolumeFile(volumeFile, preallocateSize, 0) + if err != nil { + return fmt.Errorf("create volume file %s: %v", volumeFile, err) + } + } + // println("source:", volFileInfoResp.String()) - if err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".dat", false, true); err != nil { + copyResponse := &volume_server_pb.VolumeCopyResponse{} + reportInterval := int64(1024 * 1024 * 128) + nextReportTarget := reportInterval + var modifiedTsNs int64 + var sendErr error + if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".dat", false, true, func(processed int64) bool { + if processed > nextReportTarget { + copyResponse.ProcessedBytes = processed + if sendErr = stream.Send(copyResponse); sendErr != nil { + return false + } + nextReportTarget = processed + reportInterval + } + return true + }); err != nil { return err } + if sendErr != nil { + return sendErr + } + if modifiedTsNs > 0 { + os.Chtimes(dataBaseFileName+".dat", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs)) + } - if err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, indexBaseFileName, ".idx", false, false); err != nil { + if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, indexBaseFileName, ".idx", false, false, nil); err != nil { return err } + if modifiedTsNs > 0 { + os.Chtimes(indexBaseFileName+".idx", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs)) + } - if err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".vif", false, true); err != nil { + if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".vif", false, true, nil); err != nil { return err } + if modifiedTsNs > 0 { + os.Chtimes(dataBaseFileName+".vif", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs)) + } os.Remove(dataBaseFileName + ".note") @@ -97,10 +147,10 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo }) if err != nil { - return nil, err + return err } if dataBaseFileName == "" { - return nil, fmt.Errorf("not found volume %d file", req.VolumeId) + return fmt.Errorf("not found volume %d file", req.VolumeId) } idxFileName = indexBaseFileName + ".idx" @@ -115,21 +165,25 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo }() if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16 - return nil, err + return err } // mount the volume err = vs.store.MountVolume(needle.VolumeId(req.VolumeId)) if err != nil { - return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err) + return fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err) } - return &volume_server_pb.VolumeCopyResponse{ + if err = stream.Send(&volume_server_pb.VolumeCopyResponse{ LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second), - }, err + }); err != nil { + glog.Errorf("send response: %v", err) + } + + return err } -func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool) error { +func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool, progressFn storage.ProgressFunc) (modifiedTsNs int64, err error) { copyFileClient, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{ VolumeId: vid, @@ -141,15 +195,15 @@ func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, i IgnoreSourceFileNotFound: ignoreSourceFileNotFound, }) if err != nil { - return fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err) + return modifiedTsNs, fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err) } - err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend) + modifiedTsNs, err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend, progressFn) if err != nil { - return fmt.Errorf("failed to copy %s file: %v", baseFileName+ext, err) + return modifiedTsNs, fmt.Errorf("failed to copy %s file: %v", baseFileName+ext, err) } - return nil + return modifiedTsNs, nil } @@ -178,7 +232,7 @@ func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse return nil } -func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool) error { +func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool, progressFn storage.ProgressFunc) (modifiedTsNs int64, err error) { glog.V(4).Infof("writing to %s", fileName) flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC if isAppend { @@ -186,22 +240,32 @@ func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName s } dst, err := os.OpenFile(fileName, flags, 0644) if err != nil { - return nil + return modifiedTsNs, nil } defer dst.Close() + var progressedBytes int64 for { resp, receiveErr := client.Recv() if receiveErr == io.EOF { break } + if resp != nil && resp.ModifiedTsNs != 0 { + modifiedTsNs = resp.ModifiedTsNs + } if receiveErr != nil { - return fmt.Errorf("receiving %s: %v", fileName, receiveErr) + return modifiedTsNs, fmt.Errorf("receiving %s: %v", fileName, receiveErr) } dst.Write(resp.FileContent) + progressedBytes += int64(len(resp.FileContent)) + if progressFn != nil { + if !progressFn(progressedBytes) { + return modifiedTsNs, fmt.Errorf("interrupted copy operation") + } + } wt.MaybeSlowdown(int64(len(resp.FileContent))) } - return nil + return modifiedTsNs, nil } func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) { @@ -239,6 +303,7 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v if uint32(v.CompactionRevision) != req.CompactionRevision && req.CompactionRevision != math.MaxUint32 { return fmt.Errorf("volume %d is compacted", req.VolumeId) } + v.SyncToDisk() fileName = v.FileName(req.Ext) } else { baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) + req.Ext @@ -271,6 +336,12 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v } defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + return err + } + fileModTsNs := fileInfo.ModTime().UnixNano() + buffer := make([]byte, BufferSizeLimit) for bytesToRead > 0 { @@ -290,12 +361,14 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v bytesread = int(bytesToRead) } err = stream.Send(&volume_server_pb.CopyFileResponse{ - FileContent: buffer[:bytesread], + FileContent: buffer[:bytesread], + ModifiedTsNs: fileModTsNs, }) if err != nil { // println("sending", bytesread, "bytes err", err.Error()) return err } + fileModTsNs = 0 // only send once bytesToRead -= int64(bytesread) diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index d7e4f302a..79611f499 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -3,9 +3,7 @@ package weed_server import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/volume_info" "io" - "io/ioutil" "math" "os" "path" @@ -13,11 +11,13 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/storage/volume_info" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -49,6 +49,17 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_ return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection) } + shouldCleanup := true + defer func() { + if !shouldCleanup { + return + } + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + os.Remove(fmt.Sprintf("%s.ec%2d", baseFileName, i)) + } + os.Remove(v.IndexFileName() + ".ecx") + }() + // write .ec00 ~ .ec13 files if err := erasure_coding.WriteEcFiles(baseFileName); err != nil { return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err) @@ -64,6 +75,8 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_ return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err) } + shouldCleanup = false + return &volume_server_pb.VolumeEcShardsGenerateResponse{}, nil } @@ -113,11 +126,11 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv dataBaseFileName := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId)) indexBaseFileName := storage.VolumeFileName(location.IdxDirectory, req.Collection, int(req.VolumeId)) - err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(true, pb.ServerAddress(req.SourceDataNode), vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { // copy ec data slices for _, shardId := range req.ShardIds { - if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false); err != nil { + if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false, nil); err != nil { return err } } @@ -125,7 +138,7 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv if req.CopyEcxFile { // copy ecx file - if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false); err != nil { + if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false, nil); err != nil { return err } return nil @@ -133,14 +146,14 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv if req.CopyEcjFile { // copy ecj file - if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecj", true, true); err != nil { + if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecj", true, true, nil); err != nil { return err } } if req.CopyVifFile { // copy vif file - if err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true); err != nil { + if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil { return err } } @@ -186,12 +199,12 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se existingShardCount := 0 for _, location := range vs.store.Locations { - fileInfos, err := ioutil.ReadDir(location.Directory) + fileInfos, err := os.ReadDir(location.Directory) if err != nil { continue } if location.IdxDirectory != location.Directory { - idxFileInfos, err := ioutil.ReadDir(location.IdxDirectory) + idxFileInfos, err := os.ReadDir(location.IdxDirectory) if err != nil { continue } diff --git a/weed/server/volume_grpc_read_all.go b/weed/server/volume_grpc_read_all.go new file mode 100644 index 000000000..7fe5bad03 --- /dev/null +++ b/weed/server/volume_grpc_read_all.go @@ -0,0 +1,36 @@ +package weed_server + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +func (vs *VolumeServer) ReadAllNeedles(req *volume_server_pb.ReadAllNeedlesRequest, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer) (err error) { + + for _, vid := range req.VolumeIds { + if err := vs.streaReadOneVolume(needle.VolumeId(vid), stream, err); err != nil { + return err + } + } + return nil +} + +func (vs *VolumeServer) streaReadOneVolume(vid needle.VolumeId, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer, err error) error { + v := vs.store.GetVolume(vid) + if v == nil { + return fmt.Errorf("not found volume id %d", vid) + } + + scanner := &storage.VolumeFileScanner4ReadAll{ + Stream: stream, + V: v, + } + + offset := int64(v.SuperBlock.BlockSize()) + + err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, offset, scanner) + + return err +} diff --git a/weed/server/volume_grpc_remote.go b/weed/server/volume_grpc_remote.go index 0615a96a1..9114db329 100644 --- a/weed/server/volume_grpc_remote.go +++ b/weed/server/volume_grpc_remote.go @@ -3,10 +3,14 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" + "sync" + "time" ) func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_server_pb.FetchAndWriteNeedleRequest) (resp *volume_server_pb.FetchAndWriteNeedleResponse, err error) { @@ -30,16 +34,50 @@ func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_ser return nil, fmt.Errorf("read from remote %+v: %v", remoteStorageLocation, ReadRemoteErr) } - n := new(needle.Needle) - n.Id = types.NeedleId(req.NeedleId) - n.Cookie = types.Cookie(req.Cookie) - n.Data, n.DataSize = data, uint32(len(data)) - // copied from *Needle.prepareWriteBuffer() - n.Size = 4 + types.Size(n.DataSize) + 1 - n.Checksum = needle.NewCRC(n.Data) - if _, err = vs.store.WriteVolumeNeedle(v.Id, n, true, false); err != nil { - return nil, fmt.Errorf("write needle %d size %d: %v", req.NeedleId, req.Size, err) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + n := new(needle.Needle) + n.Id = types.NeedleId(req.NeedleId) + n.Cookie = types.Cookie(req.Cookie) + n.Data, n.DataSize = data, uint32(len(data)) + // copied from *Needle.prepareWriteBuffer() + n.Size = 4 + types.Size(n.DataSize) + 1 + n.Checksum = needle.NewCRC(n.Data) + n.LastModified = uint64(time.Now().Unix()) + n.SetHasLastModifiedDate() + if _, localWriteErr := vs.store.WriteVolumeNeedle(v.Id, n, true, false); localWriteErr != nil { + if err == nil { + err = fmt.Errorf("local write needle %d size %d: %v", req.NeedleId, req.Size, err) + } + } + }() + if len(req.Replicas) > 0 { + fileId := needle.NewFileId(v.Id, req.NeedleId, req.Cookie) + for _, replica := range req.Replicas { + wg.Add(1) + go func(targetVolumeServer string) { + defer wg.Done() + uploadOption := &operation.UploadOption{ + UploadUrl: fmt.Sprintf("http://%s/%s?type=replicate", targetVolumeServer, fileId.String()), + Filename: "", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: security.EncodedJwt(req.Auth), + } + if _, replicaWriteErr := operation.UploadData(data, uploadOption); replicaWriteErr != nil { + if err == nil { + err = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, err) + } + } + }(replica.Url) + } } - return resp, nil + wg.Wait() + + return resp, err } diff --git a/weed/server/volume_grpc_tail.go b/weed/server/volume_grpc_tail.go index 3ea902ed3..4022da44a 100644 --- a/weed/server/volume_grpc_tail.go +++ b/weed/server/volume_grpc_tail.go @@ -3,6 +3,7 @@ package weed_server import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "time" "github.com/chrislusf/seaweedfs/weed/glog" @@ -89,7 +90,7 @@ func (vs *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serv defer glog.V(1).Infof("receive tailing volume %d finished", v.Id) - return resp, operation.TailVolumeFromSource(req.SourceVolumeServer, vs.grpcDialOption, v.Id, req.SinceNs, int(req.IdleTimeoutSeconds), func(n *needle.Needle) error { + return resp, operation.TailVolumeFromSource(pb.ServerAddress(req.SourceVolumeServer), vs.grpcDialOption, v.Id, req.SinceNs, int(req.IdleTimeoutSeconds), func(n *needle.Needle) error { _, err := vs.store.WriteVolumeNeedle(v.Id, n, false, false) return err }) diff --git a/weed/server/volume_grpc_tier_upload.go b/weed/server/volume_grpc_tier_upload.go index e51de5f1d..c690de959 100644 --- a/weed/server/volume_grpc_tier_upload.go +++ b/weed/server/volume_grpc_tier_upload.go @@ -27,7 +27,7 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi // locate the disk file diskFile, ok := v.DataBackend.(*backend.DiskFile) if !ok { - return fmt.Errorf("volume %d is not on local disk", req.VolumeId) + return nil // already copied to remove. fmt.Errorf("volume %d is not on local disk", req.VolumeId) } // check valid storage backend type @@ -62,13 +62,8 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi }) } - // remember the file original source - attributes := make(map[string]string) - attributes["volumeId"] = v.Id.String() - attributes["collection"] = v.Collection - attributes["ext"] = ".dat" // copy the data file - key, size, err := backendStorage.CopyFile(diskFile.File, attributes, fn) + key, size, err := backendStorage.CopyFile(diskFile.File, fn) if err != nil { return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.Name(), err) } diff --git a/weed/server/volume_grpc_vacuum.go b/weed/server/volume_grpc_vacuum.go index f8d1b7fda..0ab782b02 100644 --- a/weed/server/volume_grpc_vacuum.go +++ b/weed/server/volume_grpc_vacuum.go @@ -24,19 +24,35 @@ func (vs *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_serve } -func (vs *VolumeServer) VacuumVolumeCompact(ctx context.Context, req *volume_server_pb.VacuumVolumeCompactRequest) (*volume_server_pb.VacuumVolumeCompactResponse, error) { +func (vs *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCompactRequest, stream volume_server_pb.VolumeServer_VacuumVolumeCompactServer) error { resp := &volume_server_pb.VacuumVolumeCompactResponse{} - - err := vs.store.CompactVolume(needle.VolumeId(req.VolumeId), req.Preallocate, vs.compactionBytePerSecond) + reportInterval := int64(1024 * 1024 * 128) + nextReportTarget := reportInterval + + var sendErr error + err := vs.store.CompactVolume(needle.VolumeId(req.VolumeId), req.Preallocate, vs.compactionBytePerSecond, func(processed int64) bool { + if processed > nextReportTarget { + resp.ProcessedBytes = processed + if sendErr = stream.Send(resp); sendErr != nil { + return false + } + nextReportTarget = processed + reportInterval + } + return true + }) if err != nil { glog.Errorf("compact volume %d: %v", req.VolumeId, err) - } else { - glog.V(1).Infof("compact volume %d", req.VolumeId) + return err + } + if sendErr != nil { + glog.Errorf("compact volume %d report progress: %v", req.VolumeId, sendErr) + return sendErr } - return resp, err + glog.V(1).Infof("compact volume %d", req.VolumeId) + return nil } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 034521b4b..abb30229a 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -1,10 +1,13 @@ package weed_server import ( - "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" "net/http" "sync" + "time" + + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/types" "google.golang.org/grpc" @@ -17,15 +20,17 @@ import ( ) type VolumeServer struct { + volume_server_pb.UnimplementedVolumeServerServer inFlightUploadDataSize int64 inFlightDownloadDataSize int64 concurrentUploadLimit int64 concurrentDownloadLimit int64 inFlightUploadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond + inflightUploadDataTimeout time.Duration - SeedMasterNodes []string - currentMaster string + SeedMasterNodes []pb.ServerAddress + currentMaster pb.ServerAddress pulseSeconds int dataCenter string rack string @@ -45,11 +50,11 @@ type VolumeServer struct { } func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, - port int, publicUrl string, + port int, grpcPort int, publicUrl string, folders []string, maxCounts []int, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType, idxFolder string, needleMapKind storage.NeedleMapKind, - masterNodes []string, pulseSeconds int, + masterNodes []pb.ServerAddress, pulseSeconds int, dataCenter string, rack string, whiteList []string, fixJpgOrientation bool, @@ -58,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitMB int, concurrentUploadLimit int64, concurrentDownloadLimit int64, + inflightUploadDataTimeout time.Duration, ) *VolumeServer { v := util.GetViper() @@ -86,16 +92,18 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, + inflightUploadDataTimeout: inflightUploadDataTimeout, } vs.SeedMasterNodes = masterNodes vs.checkWithMaster() - vs.store = storage.NewStore(vs.grpcDialOption, port, ip, publicUrl, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes) + vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes) vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) handleStaticResources(adminMux) adminMux.HandleFunc("/status", vs.statusHandler) + adminMux.HandleFunc("/healthz", vs.healthzHandler) if signingKey == "" || enableUiAccess { // only expose the volume server details for safe environments adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler) @@ -113,7 +121,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, } go vs.heartbeat() - go stats.LoopPushingMetric("volumeServer", fmt.Sprintf("%s:%d", ip, port), vs.metricsAddress, vs.metricsIntervalSec) + go stats.LoopPushingMetric("volumeServer", util.JoinHostPort(ip, port), vs.metricsAddress, vs.metricsIntervalSec) return vs } diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index ff2eccc11..293f36f14 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,10 +1,12 @@ package weed_server import ( + "fmt" "net/http" "strconv" "strings" "sync/atomic" + "time" "github.com/chrislusf/seaweedfs/weed/util" @@ -39,8 +41,14 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque stats.ReadRequest() vs.inFlightDownloadDataLimitCond.L.Lock() for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit { - glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit) - vs.inFlightDownloadDataLimitCond.Wait() + select { + case <-r.Context().Done(): + glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err()) + return + default: + glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit) + vs.inFlightDownloadDataLimitCond.Wait() + } } vs.inFlightDownloadDataLimitCond.L.Unlock() vs.GetOrHeadHandler(w, r) @@ -49,18 +57,31 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque vs.guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT", "POST": - // wait until in flight data is less than the limit contentLength := getContentLength(r) - vs.inFlightUploadDataLimitCond.L.Lock() - for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { - glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) - vs.inFlightUploadDataLimitCond.Wait() + // exclude the replication from the concurrentUploadLimitMB + if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { + startTime := time.Now() + vs.inFlightUploadDataLimitCond.L.Lock() + for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { + //wait timeout check + if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { + vs.inFlightUploadDataLimitCond.L.Unlock() + err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + glog.V(1).Infof("too many requests: %v", err) + writeJsonError(w, r, http.StatusTooManyRequests, err) + return + } + glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + vs.inFlightUploadDataLimitCond.Wait() + } + vs.inFlightUploadDataLimitCond.L.Unlock() } - vs.inFlightUploadDataLimitCond.L.Unlock() atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) - vs.inFlightUploadDataLimitCond.Signal() + if vs.concurrentUploadLimit != 0 { + vs.inFlightUploadDataLimitCond.Signal() + } }() // processs uploads @@ -133,7 +154,7 @@ func (vs *VolumeServer) maybeCheckJwtAuthorization(r *http.Request, vid, fid str return false } - token, err := security.DecodeJwt(signingKey, tokenStr) + token, err := security.DecodeJwt(signingKey, tokenStr, &security.SeaweedFileIdClaims{}) if err != nil { glog.V(1).Infof("jwt verification error from %s: %v", r.RemoteAddr, err) return false diff --git a/weed/server/volume_server_handlers_admin.go b/weed/server/volume_server_handlers_admin.go index 7e6c06871..37cf109e2 100644 --- a/weed/server/volume_server_handlers_admin.go +++ b/weed/server/volume_server_handlers_admin.go @@ -1,6 +1,7 @@ package weed_server import ( + "github.com/chrislusf/seaweedfs/weed/topology" "net/http" "path/filepath" @@ -9,6 +10,24 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +func (vs *VolumeServer) healthzHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) + volumeInfos := vs.store.VolumeInfos() + for _, vinfo := range volumeInfos { + if len(vinfo.Collection) == 0 { + continue + } + if vinfo.ReplicaPlacement.GetCopyCount() > 1 { + _, err := topology.GetWritableRemoteReplications(vs.store, vs.grpcDialOption, vinfo.Id, vs.GetMaster) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + } + } + w.WriteHeader(http.StatusOK) +} + func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) m := make(map[string]interface{}) diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 5d12108d3..eb5b2be5a 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util/mem" "io" "mime" "net/http" @@ -29,8 +30,6 @@ var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { - glog.V(9).Info(r.Method + " " + r.URL.Path + " " + r.Header.Get("Range")) - stats.VolumeServerRequestCounter.WithLabelValues("get").Inc() start := time.Now() defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }() @@ -103,7 +102,9 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } w.WriteHeader(response.StatusCode) - io.Copy(w, response.Body) + buf := mem.Allocate(128 * 1024) + defer mem.Free(buf) + io.CopyBuffer(w, response.Body, buf) return } else { // redirect @@ -126,6 +127,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) var count int var needleSize types.Size + readOption.AttemptMetaOnly, readOption.MustMetaOnly = shouldAttemptStreamWrite(hasVolume, ext, r) onReadSizeFn := func(size types.Size) { needleSize = size atomic.AddInt64(&vs.inFlightDownloadDataSize, int64(needleSize)) @@ -140,7 +142,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) vs.inFlightDownloadDataLimitCond.Signal() }() - if err != nil && err != storage.ErrorDeleted && r.FormValue("type") != "replicate" && hasVolume { + if err != nil && err != storage.ErrorDeleted && hasVolume { glog.V(4).Infof("read needle: %v", err) // start to fix it from other replicas, if not deleted and hasVolume and is not a replicated request } @@ -217,11 +219,31 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } - rs := conditionallyResizeImages(bytes.NewReader(n.Data), ext, r) + if !readOption.IsMetaOnly { + rs := conditionallyResizeImages(bytes.NewReader(n.Data), ext, r) + if e := writeResponseContent(filename, mtype, rs, w, r); e != nil { + glog.V(2).Infoln("response write error:", e) + } + } else { + vs.streamWriteResponseContent(filename, mtype, volumeId, n, w, r, readOption) + } +} - if e := writeResponseContent(filename, mtype, rs, w, r); e != nil { - glog.V(2).Infoln("response write error:", e) +func shouldAttemptStreamWrite(hasLocalVolume bool, ext string, r *http.Request) (shouldAttempt bool, mustMetaOnly bool) { + if !hasLocalVolume { + return false, false + } + if len(ext) > 0 { + ext = strings.ToLower(ext) } + if r.Method == "HEAD" { + return true, true + } + _, _, _, shouldResize := shouldResizeImages(ext, r) + if shouldResize { + return false, false + } + return true, false } func (vs *VolumeServer) tryHandleChunkedFile(n *needle.Needle, fileName string, ext string, w http.ResponseWriter, r *http.Request) (processed bool) { @@ -301,7 +323,7 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re } w.Header().Set("Accept-Ranges", "bytes") - adjustHeaderContentDisposition(w, r, filename) + adjustPassthroughHeaders(w, r, filename) if r.Method == "HEAD" { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) @@ -317,3 +339,27 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re }) return nil } + +func (vs *VolumeServer) streamWriteResponseContent(filename string, mimeType string, volumeId needle.VolumeId, n *needle.Needle, w http.ResponseWriter, r *http.Request, readOption *storage.ReadOption) { + totalSize := int64(n.DataSize) + if mimeType == "" { + if ext := filepath.Ext(filename); ext != "" { + mimeType = mime.TypeByExtension(ext) + } + } + if mimeType != "" { + w.Header().Set("Content-Type", mimeType) + } + w.Header().Set("Accept-Ranges", "bytes") + adjustPassthroughHeaders(w, r, filename) + + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) + return + } + + processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { + return vs.store.ReadVolumeNeedleDataInto(volumeId, n, readOption, writer, offset, size) + }) + +} diff --git a/weed/server/volume_server_handlers_ui.go b/weed/server/volume_server_handlers_ui.go index 437e5c45d..2c420c2d6 100644 --- a/weed/server/volume_server_handlers_ui.go +++ b/weed/server/volume_server_handlers_ui.go @@ -1,6 +1,7 @@ package weed_server import ( + "github.com/chrislusf/seaweedfs/weed/pb" "net/http" "path/filepath" "time" @@ -35,7 +36,7 @@ func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) } args := struct { Version string - Masters []string + Masters []pb.ServerAddress Volumes interface{} EcVolumes interface{} RemoteVolumes interface{} diff --git a/weed/server/volume_server_tcp_handlers_write.go b/weed/server/volume_server_tcp_handlers_write.go index a009611da..24ad916e6 100644 --- a/weed/server/volume_server_tcp_handlers_write.go +++ b/weed/server/volume_server_tcp_handlers_write.go @@ -3,12 +3,13 @@ package weed_server import ( "bufio" "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage/needle" - "github.com/chrislusf/seaweedfs/weed/util" "io" "net" "strings" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/util" ) func (vs *VolumeServer) HandleTcpConnection(c net.Conn) { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 68c1f3233..265dea03a 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "math" "os" "path" "strings" @@ -27,19 +26,18 @@ import ( ) type WebDavOption struct { - Filer string - FilerGrpcAddress string - DomainName string - BucketsPath string - GrpcDialOption grpc.DialOption - Collection string - Replication string - DiskType string - Uid uint32 - Gid uint32 - Cipher bool - CacheDir string - CacheSizeMB int64 + Filer pb.ServerAddress + DomainName string + BucketsPath string + GrpcDialOption grpc.DialOption + Collection string + Replication string + DiskType string + Uid uint32 + Gid uint32 + Cipher bool + CacheDir string + CacheSizeMB int64 } type WebDavServer struct { @@ -107,7 +105,7 @@ type WebDavFile struct { func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) { - cacheUniqueId := util.Md5String([]byte("webdav" + option.FilerGrpcAddress + util.Version()))[0:8] + cacheUniqueId := util.Md5String([]byte("webdav" + string(option.Filer) + util.Version()))[0:8] cacheDir := path.Join(option.CacheDir, cacheUniqueId) os.MkdirAll(cacheDir, os.FileMode(0755)) @@ -121,12 +119,12 @@ func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) { var _ = filer_pb.FilerClient(&WebDavFileSystem{}) -func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (fs *WebDavFileSystem) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) - }, fs.option.FilerGrpcAddress, fs.option.GrpcDialOption) + }, fs.option.Filer.ToGrpcAddress(), fs.option.GrpcDialOption) } func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string { @@ -163,7 +161,7 @@ func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm return os.ErrExist } - return fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { dir, name := util.FullPath(fullDirPath).DirAndName() request := &filer_pb.CreateEntryRequest{ Directory: dir, @@ -213,21 +211,19 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f } dir, name := util.FullPath(fullFilePath).DirAndName() - err = fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ Directory: dir, Entry: &filer_pb.Entry{ Name: name, IsDirectory: perm&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(perm), - Uid: fs.option.Uid, - Gid: fs.option.Gid, - Collection: fs.option.Collection, - Replication: fs.option.Replication, - TtlSec: 0, + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(perm), + Uid: fs.option.Uid, + Gid: fs.option.Gid, + TtlSec: 0, }, }, Signatures: []int32{fs.signature}, @@ -316,7 +312,7 @@ func (fs *WebDavFileSystem) Rename(ctx context.Context, oldName, newName string) oldDir, oldBaseName := util.FullPath(oldName).DirAndName() newDir, newBaseName := util.FullPath(newName).DirAndName() - return fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AtomicRenameEntryRequest{ OldDirectory: oldDir, @@ -376,7 +372,7 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64 var fileId, host string var auth security.EncodedJwt - if flushErr := f.fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if flushErr := f.fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { ctx := context.Background() @@ -398,7 +394,7 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64 return fmt.Errorf("assign volume failure %v: %v", request, resp.Error) } - fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth) + fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth) f.collection, f.replication = resp.Collection, resp.Replication return nil @@ -413,7 +409,16 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64 } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - uploadResult, flushErr, _ := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, reader, false, "", nil, auth) + uploadOption := &operation.UploadOption{ + UploadUrl: fileUrl, + Filename: f.name, + Cipher: f.fs.option.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + } + uploadResult, flushErr, _ := operation.Upload(reader, uploadOption) if flushErr != nil { glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr) return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr) @@ -469,10 +474,8 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { f.entry.Chunks = manifestedChunks } - flushErr := f.fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + flushErr := f.fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { f.entry.Attributes.Mtime = time.Now().Unix() - f.entry.Attributes.Collection = f.collection - f.entry.Attributes.Replication = f.replication request := &filer_pb.UpdateEntryRequest{ Directory: dir, @@ -532,11 +535,11 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { return 0, io.EOF } if f.entryViewCache == nil { - f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks, 0, math.MaxInt64) + f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks, 0, fileSize) f.reader = nil } if f.reader == nil { - chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt64) + chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, fileSize) f.reader = filer.NewChunkReaderAtFromClient(filer.LookupFn(f.fs), chunkViews, f.fs.chunkCache, fileSize) } diff --git a/weed/shell/command_cluster_check.go b/weed/shell/command_cluster_check.go new file mode 100644 index 000000000..e363239e7 --- /dev/null +++ b/weed/shell/command_cluster_check.go @@ -0,0 +1,249 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandClusterCheck{}) +} + +type commandClusterCheck struct { +} + +func (c *commandClusterCheck) Name() string { + return "cluster.check" +} + +func (c *commandClusterCheck) Help() string { + return `check current cluster network connectivity + + cluster.check + +` +} + +func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if err = clusterPsCommand.Parse(args); err != nil { + return nil + } + + // collect topology information + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) + if err != nil { + return err + } + fmt.Fprintf(writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(topologyInfo.DiskInfos)) + + emptyDiskTypeDiskInfo, emptyDiskTypeFound := topologyInfo.DiskInfos[""] + hddDiskTypeDiskInfo, hddDiskTypeFound := topologyInfo.DiskInfos["hdd"] + if !emptyDiskTypeFound && !hddDiskTypeFound { + return fmt.Errorf("Need to a hdd disk type!") + } + if emptyDiskTypeFound && emptyDiskTypeDiskInfo.MaxVolumeCount == 0 || hddDiskTypeFound && hddDiskTypeDiskInfo.MaxVolumeCount == 0 { + return fmt.Errorf("Need to a hdd disk type!") + } + + // collect filers + var filers []pb.ServerAddress + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: *commandEnv.option.FilerGroup, + }) + + for _, node := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(node.Address)) + } + return err + }) + if err != nil { + return + } + fmt.Fprintf(writer, "the cluster has %d filers: %+v\n", len(filers), filers) + + // collect volume servers + var volumeServers []pb.ServerAddress + t, _, err := collectTopologyInfo(commandEnv, 0) + if err != nil { + return err + } + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn)) + } + } + } + fmt.Fprintf(writer, "the cluster has %d volume servers: %+v\n", len(volumeServers), volumeServers) + + // collect all masters + var masters []pb.ServerAddress + for _, master := range commandEnv.MasterClient.GetMasters() { + masters = append(masters, master) + } + + // check from master to volume servers + for _, master := range masters { + for _, volumeServer := range volumeServers { + fmt.Fprintf(writer, "checking master %s to volume server %s ... ", string(master), string(volumeServer)) + err := pb.WithMasterClient(false, master, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error { + pong, err := client.Ping(context.Background(), &master_pb.PingRequest{ + Target: string(volumeServer), + TargetType: cluster.VolumeServerType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check between masters + for _, sourceMaster := range masters { + for _, targetMaster := range masters { + if sourceMaster == targetMaster { + continue + } + fmt.Fprintf(writer, "checking master %s to %s ... ", string(sourceMaster), string(targetMaster)) + err := pb.WithMasterClient(false, sourceMaster, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error { + pong, err := client.Ping(context.Background(), &master_pb.PingRequest{ + Target: string(targetMaster), + TargetType: cluster.MasterType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check from volume servers to masters + for _, volumeServer := range volumeServers { + for _, master := range masters { + fmt.Fprintf(writer, "checking volume server %s to master %s ... ", string(volumeServer), string(master)) + err := pb.WithVolumeServerClient(false, volumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + pong, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{ + Target: string(master), + TargetType: cluster.MasterType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check from filers to masters + for _, filer := range filers { + for _, master := range masters { + fmt.Fprintf(writer, "checking filer %s to master %s ... ", string(filer), string(master)) + err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{ + Target: string(master), + TargetType: cluster.MasterType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check from filers to volume servers + for _, filer := range filers { + for _, volumeServer := range volumeServers { + fmt.Fprintf(writer, "checking filer %s to volume server %s ... ", string(filer), string(volumeServer)) + err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{ + Target: string(volumeServer), + TargetType: cluster.VolumeServerType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check between volume servers + for _, sourceVolumeServer := range volumeServers { + for _, targetVolumeServer := range volumeServers { + if sourceVolumeServer == targetVolumeServer { + continue + } + fmt.Fprintf(writer, "checking volume server %s to %s ... ", string(sourceVolumeServer), string(targetVolumeServer)) + err := pb.WithVolumeServerClient(false, sourceVolumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + pong, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{ + Target: string(targetVolumeServer), + TargetType: cluster.VolumeServerType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + // check between filers, and need to connect to itself + for _, sourceFiler := range filers { + for _, targetFiler := range filers { + fmt.Fprintf(writer, "checking filer %s to %s ... ", string(sourceFiler), string(targetFiler)) + err := pb.WithFilerClient(false, sourceFiler, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{ + Target: string(targetFiler), + TargetType: cluster.FilerType, + }) + if err == nil { + printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs) + } + return err + }) + if err != nil { + fmt.Fprintf(writer, "%v\n", err) + } + } + } + + return nil +} + +func printTiming(writer io.Writer, startNs, remoteNs, stopNs int64) { + roundTripTimeMs := float32(stopNs-startNs) / 1000000 + deltaTimeMs := float32(remoteNs-(startNs+stopNs)/2) / 1000000 + fmt.Fprintf(writer, "ok round trip %.3fms clock delta %.3fms\n", roundTripTimeMs, deltaTimeMs) +} diff --git a/weed/shell/command_cluster_ps.go b/weed/shell/command_cluster_ps.go new file mode 100644 index 000000000..21d3587fb --- /dev/null +++ b/weed/shell/command_cluster_ps.go @@ -0,0 +1,109 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "io" + + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" +) + +func init() { + Commands = append(Commands, &commandClusterPs{}) +} + +type commandClusterPs struct { +} + +func (c *commandClusterPs) Name() string { + return "cluster.ps" +} + +func (c *commandClusterPs) Help() string { + return `check current cluster process status + + cluster.ps + +` +} + +func (c *commandClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if err = clusterPsCommand.Parse(args); err != nil { + return nil + } + + var filerNodes []*master_pb.ListClusterNodesResponse_ClusterNode + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: *commandEnv.option.FilerGroup, + }) + if err != nil { + return err + } + + filerNodes = resp.ClusterNodes + return err + }) + if err != nil { + return + } + + fmt.Fprintf(writer, "* filers %d\n", len(filerNodes)) + for _, node := range filerNodes { + fmt.Fprintf(writer, " * %s (%v)\n", node.Address, node.Version) + pb.WithFilerClient(false, pb.ServerAddress(node.Address), commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err == nil { + if resp.FilerGroup != "" { + fmt.Fprintf(writer, " filer group: %s\n", resp.FilerGroup) + } + fmt.Fprintf(writer, " signature: %d\n", resp.Signature) + } else { + fmt.Fprintf(writer, " failed to connect: %v\n", err) + } + return err + }) + } + + // collect volume servers + var volumeServers []pb.ServerAddress + t, _, err := collectTopologyInfo(commandEnv, 0) + if err != nil { + return err + } + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn)) + } + } + } + + fmt.Fprintf(writer, "* volume servers %d\n", len(volumeServers)) + for _, dc := range t.DataCenterInfos { + fmt.Fprintf(writer, " * data center: %s\n", dc.Id) + for _, r := range dc.RackInfos { + fmt.Fprintf(writer, " * rack: %s\n", r.Id) + for _, dn := range r.DataNodeInfos { + pb.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dn), commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + resp, err := client.VolumeServerStatus(context.Background(), &volume_server_pb.VolumeServerStatusRequest{}) + if err == nil { + fmt.Fprintf(writer, " * %s (%v)\n", dn.Id, resp.Version) + } + return err + }) + } + } + } + + return nil +} diff --git a/weed/shell/command_cluster_raft_add.go b/weed/shell/command_cluster_raft_add.go new file mode 100644 index 000000000..e5f3c41c9 --- /dev/null +++ b/weed/shell/command_cluster_raft_add.go @@ -0,0 +1,59 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandRaftServerAdd{}) +} + +type commandRaftServerAdd struct { +} + +func (c *commandRaftServerAdd) Name() string { + return "cluster.raft.add" +} + +func (c *commandRaftServerAdd) Help() string { + return `add a server to the raft cluster + + Example: + cluster.raft.add -id -address -voter +` +} + +func (c *commandRaftServerAdd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + serverId := raftServerAddCommand.String("id", "", "server id") + serverAddress := raftServerAddCommand.String("address", "", "server grpc address") + serverVoter := raftServerAddCommand.Bool("voter", true, "assign it a vote") + if err = raftServerAddCommand.Parse(args); err != nil { + return nil + } + + if *serverId == "" || *serverAddress == "" { + return fmt.Errorf("empty server id or address") + } + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + _, err := client.RaftAddServer(context.Background(), &master_pb.RaftAddServerRequest{ + Id: *serverId, + Address: *serverAddress, + Voter: *serverVoter, + }) + if err != nil { + return fmt.Errorf("raft add server: %v", err) + } + println("added server", *serverId) + return nil + }) + + return err + +} diff --git a/weed/shell/command_cluster_raft_ps.go b/weed/shell/command_cluster_raft_ps.go new file mode 100644 index 000000000..ea868db06 --- /dev/null +++ b/weed/shell/command_cluster_raft_ps.go @@ -0,0 +1,51 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandRaftClusterPs{}) +} + +type commandRaftClusterPs struct { +} + +func (c *commandRaftClusterPs) Name() string { + return "cluster.raft.ps" +} + +func (c *commandRaftClusterPs) Help() string { + return `check current raft cluster status + + cluster.raft.ps +` +} + +func (c *commandRaftClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + raftClusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if err = raftClusterPsCommand.Parse(args); err != nil { + return nil + } + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + resp, err := client.RaftListClusterServers(context.Background(), &master_pb.RaftListClusterServersRequest{}) + if err != nil { + return fmt.Errorf("raft list cluster: %v", err) + } + fmt.Fprintf(writer, "the raft cluster has %d servers\n", len(resp.ClusterServers)) + for _, server := range resp.ClusterServers { + fmt.Fprintf(writer, " * %s %s (%s)\n", server.Id, server.Address, server.Suffrage) + } + + return nil + }) + + return err + +} diff --git a/weed/shell/command_cluster_raft_remove.go b/weed/shell/command_cluster_raft_remove.go new file mode 100644 index 000000000..532a1469c --- /dev/null +++ b/weed/shell/command_cluster_raft_remove.go @@ -0,0 +1,56 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandRaftServerRemove{}) +} + +type commandRaftServerRemove struct { +} + +func (c *commandRaftServerRemove) Name() string { + return "cluster.raft.remove" +} + +func (c *commandRaftServerRemove) Help() string { + return `remove a server from the raft cluster + + Example: + cluster.raft.remove -id +` +} + +func (c *commandRaftServerRemove) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + serverId := raftServerAddCommand.String("id", "", "server id") + if err = raftServerAddCommand.Parse(args); err != nil { + return nil + } + + if *serverId == "" { + return fmt.Errorf("empty server id") + } + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + _, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{ + Id: *serverId, + Force: true, + }) + if err != nil { + return fmt.Errorf("raft remove server: %v", err) + } + println("removed server", *serverId) + return nil + }) + + return err + +} diff --git a/weed/shell/command_collection_delete.go b/weed/shell/command_collection_delete.go index e43f2a093..b89fe7fc0 100644 --- a/weed/shell/command_collection_delete.go +++ b/weed/shell/command_collection_delete.go @@ -29,16 +29,17 @@ func (c *commandCollectionDelete) Help() string { func (c *commandCollectionDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - colDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) collectionName := colDeleteCommand.String("collection", "", "collection to delete. Use '_default_' for the empty-named collection.") applyBalancing := colDeleteCommand.Bool("force", false, "apply the collection") if err = colDeleteCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyBalancing, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } if *collectionName == "" { return fmt.Errorf("empty collection name is not allowed") @@ -53,7 +54,7 @@ func (c *commandCollectionDelete) Do(args []string, commandEnv *CommandEnv, writ return nil } - err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err = client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ Name: *collectionName, }) diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go index ba502a6b9..47a4da553 100644 --- a/weed/shell/command_collection_list.go +++ b/weed/shell/command_collection_list.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage/super_block" "io" ) @@ -23,10 +24,10 @@ func (c *commandCollectionList) Help() string { } type CollectionInfo struct { - FileCount uint64 - DeleteCount uint64 - DeletedByteCount uint64 - Size uint64 + FileCount float64 + DeleteCount float64 + DeletedByteCount float64 + Size float64 VolumeCount int } @@ -38,21 +39,21 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer return err } - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } collectionInfos := make(map[string]*CollectionInfo) - writeCollectionInfo(writer, topologyInfo, collectionInfos) + collectCollectionInfo(topologyInfo, collectionInfos) for _, c := range collections { cif, found := collectionInfos[c] if !found { continue } - fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%d\tfileCount:%d\tdeletedBytes:%d\tdeletion:%d\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount) + fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%.0f\tfileCount:%.0f\tdeletedBytes:%.0f\tdeletion:%.0f\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount) } fmt.Fprintf(writer, "Total %d collections.\n", len(collections)) @@ -62,7 +63,7 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer func ListCollectionNames(commandEnv *CommandEnv, includeNormalVolumes, includeEcVolumes bool) (collections []string, err error) { var resp *master_pb.CollectionListResponse - err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { resp, err = client.CollectionList(context.Background(), &master_pb.CollectionListRequest{ IncludeNormalVolumes: includeNormalVolumes, IncludeEcVolumes: includeEcVolumes, @@ -85,14 +86,16 @@ func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb. cif = &CollectionInfo{} collectionInfos[c] = cif } - cif.Size += vif.Size - cif.DeleteCount += vif.DeleteCount - cif.FileCount += vif.FileCount - cif.DeletedByteCount += vif.DeletedByteCount + replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(vif.ReplicaPlacement)) + copyCount := float64(replicaPlacement.GetCopyCount()) + cif.Size += float64(vif.Size) / copyCount + cif.DeleteCount += float64(vif.DeleteCount) / copyCount + cif.FileCount += float64(vif.FileCount) / copyCount + cif.DeletedByteCount += float64(vif.DeletedByteCount) / copyCount cif.VolumeCount++ } -func writeCollectionInfo(writer io.Writer, t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { +func collectCollectionInfo(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { for _, dc := range t.DataCenterInfos { for _, r := range dc.RackInfos { for _, dn := range r.DataNodeInfos { diff --git a/weed/shell/command_ec_balance.go b/weed/shell/command_ec_balance.go index b1ca926d5..5641c6954 100644 --- a/weed/shell/command_ec_balance.go +++ b/weed/shell/command_ec_balance.go @@ -3,12 +3,12 @@ package shell import ( "flag" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "io" - "sort" - + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" + "io" ) func init() { @@ -99,10 +99,6 @@ func (c *commandEcBalance) Help() string { func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or \"EACH_COLLECTION\" for each collection") dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter") @@ -110,6 +106,11 @@ func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.W if err = balanceCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyBalancing, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } // collect all ec nodes allEcNodes, totalFreeEcSlots, err := collectEcNodes(commandEnv, *dc) @@ -215,10 +216,10 @@ func doDeduplicateEcShards(commandEnv *CommandEnv, collection string, vid needle duplicatedShardIds := []uint32{uint32(shardId)} for _, ecNode := range ecNodes[1:] { - if err := unmountEcShards(commandEnv.option.GrpcDialOption, vid, ecNode.info.Id, duplicatedShardIds); err != nil { + if err := unmountEcShards(commandEnv.option.GrpcDialOption, vid, pb.NewServerAddressFromDataNode(ecNode.info), duplicatedShardIds); err != nil { return err } - if err := sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, ecNode.info.Id, duplicatedShardIds); err != nil { + if err := sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, pb.NewServerAddressFromDataNode(ecNode.info), duplicatedShardIds); err != nil { return err } ecNode.deleteEcVolumeShards(vid, duplicatedShardIds) @@ -410,8 +411,8 @@ func doBalanceEcRack(commandEnv *CommandEnv, ecRack *EcRack, applyBalancing bool hasMove := true for hasMove { hasMove = false - sort.Slice(rackEcNodes, func(i, j int) bool { - return rackEcNodes[i].freeEcSlot > rackEcNodes[j].freeEcSlot + slices.SortFunc(rackEcNodes, func(a, b *EcNode) bool { + return a.freeEcSlot > b.freeEcSlot }) emptyNode, fullNode := rackEcNodes[0], rackEcNodes[len(rackEcNodes)-1] emptyNodeShardCount, fullNodeShardCount := ecNodeIdToShardCount[emptyNode.info.Id], ecNodeIdToShardCount[fullNode.info.Id] @@ -491,8 +492,8 @@ func pickNEcShardsToMoveFrom(ecNodes []*EcNode, vid needle.VolumeId, n int) map[ }) } } - sort.Slice(candidateEcNodes, func(i, j int) bool { - return candidateEcNodes[i].shardCount > candidateEcNodes[j].shardCount + slices.SortFunc(candidateEcNodes, func(a, b *CandidateEcNode) bool { + return a.shardCount > b.shardCount }) for i := 0; i < n; i++ { selectedEcNodeIndex := -1 diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index fd35bb14b..27b650731 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -3,17 +3,17 @@ package shell import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "math" - "sort" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "google.golang.org/grpc" + "math" ) func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, applyBalancing bool) (err error) { @@ -22,20 +22,22 @@ func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, if applyBalancing { + existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.info) + // ask destination node to copy shard and the ecx file from source node, and mount it - copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingLocation.info.Id) + copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingServerAddress) if err != nil { return err } // unmount the to be deleted shards - err = unmountEcShards(commandEnv.option.GrpcDialOption, vid, existingLocation.info.Id, copiedShardIds) + err = unmountEcShards(commandEnv.option.GrpcDialOption, vid, existingServerAddress, copiedShardIds) if err != nil { return err } // ask source node to delete the shard, and maybe the ecx file - err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, existingLocation.info.Id, copiedShardIds) + err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, existingServerAddress, copiedShardIds) if err != nil { return err } @@ -53,13 +55,14 @@ func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer *EcNode, shardIdsToCopy []uint32, - volumeId needle.VolumeId, collection string, existingLocation string) (copiedShardIds []uint32, err error) { + volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress) (copiedShardIds []uint32, err error) { fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id) - err = operation.WithVolumeServerClient(targetServer.info.Id, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + targetAddress := pb.NewServerAddressFromDataNode(targetServer.info) + err = operation.WithVolumeServerClient(false, targetAddress, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - if targetServer.info.Id != existingLocation { + if targetAddress != existingLocation { fmt.Printf("copy %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id) _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ @@ -69,7 +72,7 @@ func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption, CopyEcxFile: true, CopyEcjFile: true, CopyVifFile: true, - SourceDataNode: existingLocation, + SourceDataNode: string(existingLocation), }) if copyErr != nil { return fmt.Errorf("copy %d.%v %s => %s : %v\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id, copyErr) @@ -86,7 +89,7 @@ func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption, return fmt.Errorf("mount %d.%v on %s : %v\n", volumeId, shardIdsToCopy, targetServer.info.Id, mountErr) } - if targetServer.info.Id != existingLocation { + if targetAddress != existingLocation { copiedShardIds = shardIdsToCopy glog.V(0).Infof("%s ec volume %d deletes shards %+v", existingLocation, volumeId, copiedShardIds) } @@ -112,14 +115,14 @@ func eachDataNode(topo *master_pb.TopologyInfo, fn func(dc string, rack RackId, } func sortEcNodesByFreeslotsDecending(ecNodes []*EcNode) { - sort.Slice(ecNodes, func(i, j int) bool { - return ecNodes[i].freeEcSlot > ecNodes[j].freeEcSlot + slices.SortFunc(ecNodes, func(a, b *EcNode) bool { + return a.freeEcSlot > b.freeEcSlot }) } func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) { - sort.Slice(ecNodes, func(i, j int) bool { - return ecNodes[i].freeEcSlot < ecNodes[j].freeEcSlot + slices.SortFunc(ecNodes, func(a, b *EcNode) bool { + return a.freeEcSlot < b.freeEcSlot }) } @@ -168,7 +171,7 @@ func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (c if diskInfo == nil { return 0 } - return int(diskInfo.MaxVolumeCount-diskInfo.ActiveVolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos) + return int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos) } type RackId string @@ -202,7 +205,7 @@ func collectEcNodes(commandEnv *CommandEnv, selectedDataCenter string) (ecNodes // list all possible locations // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } @@ -233,11 +236,11 @@ func collectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter return } -func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation string, toBeDeletedShardIds []uint32) error { +func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeDeletedShardIds []uint32) error { fmt.Printf("delete %d.%v from %s\n", volumeId, toBeDeletedShardIds, sourceLocation) - return operation.WithVolumeServerClient(sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, deleteErr := volumeServerClient.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{ VolumeId: uint32(volumeId), Collection: collection, @@ -248,11 +251,11 @@ func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection strin } -func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation string, toBeUnmountedhardIds []uint32) error { +func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeUnmountedhardIds []uint32) error { fmt.Printf("unmount %d.%v from %s\n", volumeId, toBeUnmountedhardIds, sourceLocation) - return operation.WithVolumeServerClient(sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, deleteErr := volumeServerClient.VolumeEcShardsUnmount(context.Background(), &volume_server_pb.VolumeEcShardsUnmountRequest{ VolumeId: uint32(volumeId), ShardIds: toBeUnmountedhardIds, @@ -261,11 +264,11 @@ func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, s }) } -func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation string, toBeMountedhardIds []uint32) error { +func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeMountedhardIds []uint32) error { fmt.Printf("mount %d.%v on %s\n", volumeId, toBeMountedhardIds, sourceLocation) - return operation.WithVolumeServerClient(sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ VolumeId: uint32(volumeId), Collection: collection, diff --git a/weed/shell/command_ec_decode.go b/weed/shell/command_ec_decode.go index e4d597d84..59c5bf83a 100644 --- a/weed/shell/command_ec_decode.go +++ b/weed/shell/command_ec_decode.go @@ -4,8 +4,11 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/types" "io" + "time" "google.golang.org/grpc" @@ -36,26 +39,38 @@ func (c *commandEcDecode) Help() string { } func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeId := encodeCommand.Int("volumeId", 0, "the volume id") collection := encodeCommand.String("collection", "", "the collection name") + forceChanges := encodeCommand.Bool("force", false, "force the encoding even if the cluster has less than recommended 4 nodes") if err = encodeCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *forceChanges, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } vid := needle.VolumeId(*volumeId) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } + if !*forceChanges { + var nodeCount int + eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { + nodeCount++ + }) + if nodeCount < erasure_coding.ParityShardsCount { + glog.V(0).Infof("skip erasure coding with %d nodes, less than recommended %d nodes", nodeCount, erasure_coding.ParityShardsCount) + return nil + } + } + // volumeId is provided if vid != 0 { return doEcDecode(commandEnv, topologyInfo, *collection, vid) @@ -100,10 +115,10 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec return nil } -func mountVolumeAndDeleteEcShards(grpcDialOption grpc.DialOption, collection, targetNodeLocation string, nodeToEcIndexBits map[string]erasure_coding.ShardBits, vid needle.VolumeId) error { +func mountVolumeAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, targetNodeLocation pb.ServerAddress, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, vid needle.VolumeId) error { // mount volume - if err := operation.WithVolumeServerClient(targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + if err := operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{ VolumeId: uint32(vid), }) @@ -132,11 +147,11 @@ func mountVolumeAndDeleteEcShards(grpcDialOption grpc.DialOption, collection, ta return nil } -func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer string) error { +func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer) - err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{ VolumeId: uint32(vid), Collection: collection, @@ -148,7 +163,7 @@ func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, c } -func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[string]erasure_coding.ShardBits, collection string, vid needle.VolumeId) (targetNodeLocation string, err error) { +func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, collection string, vid needle.VolumeId) (targetNodeLocation pb.ServerAddress, err error) { maxShardCount := 0 var exisitngEcIndexBits erasure_coding.ShardBits @@ -174,7 +189,7 @@ func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[string]erasur continue } - err = operation.WithVolumeServerClient(targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation) @@ -185,7 +200,7 @@ func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[string]erasur CopyEcxFile: false, CopyEcjFile: true, CopyVifFile: true, - SourceDataNode: loc, + SourceDataNode: string(loc), }) if copyErr != nil { return fmt.Errorf("copy %d.%v %s => %s : %v\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation, copyErr) @@ -208,10 +223,26 @@ func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[string]erasur } -func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { +func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) { + var resp *master_pb.LookupVolumeResponse + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + resp, err = client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: volumeIds}) + return err + }) + if err != nil { + return nil, err + } + return resp.VolumeIdLocations, nil +} + +func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { + + if delayBeforeCollecting > 0 { + time.Sleep(delayBeforeCollecting) + } var resp *master_pb.VolumeListResponse - err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{}) return err }) @@ -243,14 +274,14 @@ func collectEcShardIds(topoInfo *master_pb.TopologyInfo, selectedCollection stri return } -func collectEcNodeShardBits(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[string]erasure_coding.ShardBits { +func collectEcNodeShardBits(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits { - nodeToEcIndexBits := make(map[string]erasure_coding.ShardBits) + nodeToEcIndexBits := make(map[pb.ServerAddress]erasure_coding.ShardBits) eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found { for _, v := range diskInfo.EcShardInfos { if v.Id == uint32(vid) { - nodeToEcIndexBits[dn.Id] = erasure_coding.ShardBits(v.EcIndexBits) + nodeToEcIndexBits[pb.NewServerAddressFromDataNode(dn)] = erasure_coding.ShardBits(v.EcIndexBits) } } } diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index 014b9bab7..251448908 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -4,7 +4,9 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" + "math/rand" "sync" "time" @@ -32,7 +34,7 @@ func (c *commandEcEncode) Name() string { func (c *commandEcEncode) Help() string { return `apply erasure coding to a volume - ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h] + ec.encode [-collection=""] [-fullPercent=95 -quietFor=1h] ec.encode [-collection=""] [-volumeId=] This command will: @@ -54,10 +56,6 @@ func (c *commandEcEncode) Help() string { func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeId := encodeCommand.Int("volumeId", 0, "the volume id") collection := encodeCommand.String("collection", "", "the collection name") @@ -68,6 +66,10 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + vid := needle.VolumeId(*volumeId) // volumeId is provided @@ -93,7 +95,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr func doEcEncode(commandEnv *CommandEnv, collection string, vid needle.VolumeId, parallelCopy bool) (err error) { // find volume location locations, found := commandEnv.MasterClient.GetLocations(uint32(vid)) - if !found { + if !found && len(locations) > 0 { return fmt.Errorf("volume %d not found", vid) } @@ -106,7 +108,7 @@ func doEcEncode(commandEnv *CommandEnv, collection string, vid needle.VolumeId, } // generate ec shards - err = generateEcShards(commandEnv.option.GrpcDialOption, vid, collection, locations[0].Url) + err = generateEcShards(commandEnv.option.GrpcDialOption, vid, collection, locations[0].ServerAddress()) if err != nil { return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, locations[0].Url, err) } @@ -120,11 +122,11 @@ func doEcEncode(commandEnv *CommandEnv, collection string, vid needle.VolumeId, return nil } -func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string) error { +func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { fmt.Printf("generateEcShards %s %d on %s ...\n", collection, volumeId, sourceVolumeServer) - err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, genErr := volumeServerClient.VolumeEcShardsGenerate(context.Background(), &volume_server_pb.VolumeEcShardsGenerateRequest{ VolumeId: uint32(volumeId), Collection: collection, @@ -161,13 +163,13 @@ func spreadEcShards(commandEnv *CommandEnv, volumeId needle.VolumeId, collection } // unmount the to be deleted shards - err = unmountEcShards(commandEnv.option.GrpcDialOption, volumeId, existingLocations[0].Url, copiedShardIds) + err = unmountEcShards(commandEnv.option.GrpcDialOption, volumeId, existingLocations[0].ServerAddress(), copiedShardIds) if err != nil { return err } // ask the source volume server to clean up copied ec shards - err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, existingLocations[0].Url, copiedShardIds) + err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, existingLocations[0].ServerAddress(), copiedShardIds) if err != nil { return fmt.Errorf("source delete copied ecShards %s %d.%v: %v", existingLocations[0].Url, volumeId, copiedShardIds, err) } @@ -175,7 +177,7 @@ func spreadEcShards(commandEnv *CommandEnv, volumeId needle.VolumeId, collection // ask the source volume server to delete the original volume for _, location := range existingLocations { fmt.Printf("delete volume %d from %s\n", volumeId, location.Url) - err = deleteVolume(commandEnv.option.GrpcDialOption, volumeId, location.Url) + err = deleteVolume(commandEnv.option.GrpcDialOption, volumeId, location.ServerAddress()) if err != nil { return fmt.Errorf("deleteVolume %s volume %d: %v", location.Url, volumeId, err) } @@ -194,7 +196,7 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer copyFunc := func(server *EcNode, allocatedEcShardIds []uint32) { defer wg.Done() copiedShardIds, copyErr := oneServerCopyAndMountEcShardsFromSource(grpcDialOption, server, - allocatedEcShardIds, volumeId, collection, existingLocation.Url) + allocatedEcShardIds, volumeId, collection, existingLocation.ServerAddress()) if copyErr != nil { err = copyErr } else { @@ -202,6 +204,14 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer server.addEcVolumeShards(volumeId, collection, copiedShardIds) } } + cleanupFunc := func(server *EcNode, allocatedEcShardIds []uint32) { + if err := unmountEcShards(grpcDialOption, volumeId, pb.NewServerAddressFromDataNode(server.info), allocatedEcShardIds); err != nil { + fmt.Printf("unmount aborted shards %d.%v on %s: %v\n", volumeId, allocatedEcShardIds, server.info.Id, err) + } + if err := sourceServerDeleteEcShards(grpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(server.info), allocatedEcShardIds); err != nil { + fmt.Printf("remove aborted shards %d.%v on %s: %v\n", volumeId, allocatedEcShardIds, server.info.Id, err) + } + } // maybe parallelize for i, server := range targetServers { @@ -220,6 +230,12 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer close(shardIdChan) if err != nil { + for i, server := range targetServers { + if len(allocatedEcIds[i]) <= 0 { + continue + } + cleanupFunc(server, allocatedEcIds[i]) + } return nil, err } @@ -233,7 +249,7 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) { allocated = make([][]uint32, len(servers)) allocatedShardIdIndex := uint32(0) - serverIndex := 0 + serverIndex := rand.Intn(len(servers)) for allocatedShardIdIndex < erasure_coding.TotalShardsCount { if servers[serverIndex].freeEcSlot > 0 { allocated[serverIndex] = append(allocated[serverIndex], allocatedShardIdIndex) @@ -251,7 +267,7 @@ func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) { func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) { // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } diff --git a/weed/shell/command_ec_encode_test.go b/weed/shell/command_ec_encode_test.go new file mode 100644 index 000000000..940c64266 --- /dev/null +++ b/weed/shell/command_ec_encode_test.go @@ -0,0 +1,31 @@ +package shell + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "testing" +) + +func TestEcDistribution(t *testing.T) { + + topologyInfo := parseOutput(topoData) + + // find out all volume servers with one slot left. + ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "") + + sortEcNodesByFreeslotsDecending(ecNodes) + + if totalFreeEcSlots < erasure_coding.TotalShardsCount { + println("not enough free ec shard slots", totalFreeEcSlots) + } + allocatedDataNodes := ecNodes + if len(allocatedDataNodes) > erasure_coding.TotalShardsCount { + allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount] + } + + for _, dn := range allocatedDataNodes { + // fmt.Printf("info %+v %+v\n", dn.info, dn) + fmt.Printf("=> %+v %+v\n", dn.info.Id, dn.freeEcSlot) + } + +} diff --git a/weed/shell/command_ec_rebuild.go b/weed/shell/command_ec_rebuild.go index 8d5d7bb91..a2e1f4d44 100644 --- a/weed/shell/command_ec_rebuild.go +++ b/weed/shell/command_ec_rebuild.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/operation" @@ -56,16 +57,17 @@ func (c *commandEcRebuild) Help() string { func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - fixCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) collection := fixCommand.String("collection", "EACH_COLLECTION", "collection name, or \"EACH_COLLECTION\" for each collection") applyChanges := fixCommand.Bool("force", false, "apply the changes") if err = fixCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyChanges, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } // collect all ec nodes allEcNodes, _, err := collectEcNodes(commandEnv, "") @@ -141,7 +143,7 @@ func rebuildOneEcVolume(commandEnv *CommandEnv, rebuilder *EcNode, collection st // clean up working files // ask the rebuilder to delete the copied shards - err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, rebuilder.info.Id, copiedShardIds) + err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), copiedShardIds) if err != nil { fmt.Fprintf(writer, "%s delete copied ec shards %s %d.%v\n", rebuilder.info.Id, collection, volumeId, copiedShardIds) } @@ -153,13 +155,13 @@ func rebuildOneEcVolume(commandEnv *CommandEnv, rebuilder *EcNode, collection st } // generate ec shards, and maybe ecx file - generatedShardIds, err = generateMissingShards(commandEnv.option.GrpcDialOption, collection, volumeId, rebuilder.info.Id) + generatedShardIds, err = generateMissingShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info)) if err != nil { return err } // mount the generated shards - err = mountEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, rebuilder.info.Id, generatedShardIds) + err = mountEcShards(commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), generatedShardIds) if err != nil { return err } @@ -169,9 +171,9 @@ func rebuildOneEcVolume(commandEnv *CommandEnv, rebuilder *EcNode, collection st return nil } -func generateMissingShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation string) (rebuiltShardIds []uint32, err error) { +func generateMissingShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []uint32, err error) { - err = operation.WithVolumeServerClient(sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, rebultErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{ VolumeId: uint32(volumeId), Collection: collection, @@ -212,7 +214,7 @@ func prepareDataToRecover(commandEnv *CommandEnv, rebuilder *EcNode, collection var copyErr error if applyBalancing { - copyErr = operation.WithVolumeServerClient(rebuilder.info.Id, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + copyErr = operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.info), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ VolumeId: uint32(volumeId), Collection: collection, diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go index a5731240d..16be25ee5 100644 --- a/weed/shell/command_fs_cat.go +++ b/weed/shell/command_fs_cat.go @@ -2,12 +2,10 @@ package shell import ( "fmt" - "io" - "math" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" + "io" ) func init() { @@ -41,7 +39,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write dir, name := util.FullPath(path).DirAndName() - return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Name: name, @@ -57,7 +55,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write return err } - return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry))) }) diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 0aae51d74..2c4ef2211 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -55,6 +55,9 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") isReadOnly := fsConfigureCommand.Bool("readOnly", false, "disable writes") + dataCenter := fsConfigureCommand.String("dataCenter", "", "assign writes to this dataCenter") + rack := fsConfigureCommand.String("rack", "", "assign writes to this rack") + dataNode := fsConfigureCommand.String("dataNode", "", "assign writes to this dataNode") volumeGrowthCount := fsConfigureCommand.Int("volumeGrowthCount", 0, "the number of physical volumes to add if no writable volumes") isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix") apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration") @@ -62,12 +65,13 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io return nil } - fc, err := readFilerConf(commandEnv) + fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) if err != nil { return err } if *locationPrefix != "" { + infoAboutSimulationMode(writer, *apply, "-apply") locConf := &filer_pb.FilerConf_PathConf{ LocationPrefix: *locationPrefix, Collection: *collection, @@ -77,6 +81,9 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io DiskType: *diskType, VolumeGrowthCount: uint32(*volumeGrowthCount), ReadOnly: *isReadOnly, + DataCenter: *dataCenter, + Rack: *rack, + DataNode: *dataNode, } // check collection @@ -111,7 +118,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io if *apply { - if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return filer.SaveInsideFiler(client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes()) }); err != nil && err != filer_pb.ErrNotFound { return err @@ -123,19 +130,9 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io } -func readFilerConf(commandEnv *CommandEnv) (*filer.FilerConf, error) { - var buf bytes.Buffer - if err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - return filer.ReadEntry(commandEnv.MasterClient, client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, &buf) - }); err != nil && err != filer_pb.ErrNotFound { - return nil, fmt.Errorf("read %s/%s: %v", filer.DirectoryEtcSeaweedFS, filer.FilerConfName, err) +func infoAboutSimulationMode(writer io.Writer, forceMode bool, forceModeOption string) { + if forceMode { + return } - - fc := filer.NewFilerConf() - if buf.Len() > 0 { - if err := fc.LoadFromBytes(buf.Bytes()); err != nil { - return nil, fmt.Errorf("parse %s/%s: %v", filer.DirectoryEtcSeaweedFS, filer.FilerConfName, err) - } - } - return fc, nil + fmt.Fprintf(writer, "Running in simulation mode. Use \"%s\" option to apply the changes.\n", forceModeOption) } diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go index e0525defa..4616c072d 100644 --- a/weed/shell/command_fs_meta_cat.go +++ b/weed/shell/command_fs_meta_cat.go @@ -2,11 +2,10 @@ package shell import ( "fmt" + "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + "golang.org/x/exp/slices" "io" - "sort" - - "github.com/golang/protobuf/jsonpb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -40,7 +39,7 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W dir, name := util.FullPath(path).DirAndName() - return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Name: name, @@ -55,14 +54,12 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W EmitDefaults: true, Indent: " ", } - - sort.Slice(respLookupEntry.Entry.Chunks, func(i, j int) bool { - if respLookupEntry.Entry.Chunks[i].Offset == respLookupEntry.Entry.Chunks[j].Offset { - return respLookupEntry.Entry.Chunks[i].Mtime < respLookupEntry.Entry.Chunks[j].Mtime + slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Offset == b.Offset { + return a.Mtime < b.Mtime } - return respLookupEntry.Entry.Chunks[i].Offset < respLookupEntry.Entry.Chunks[j].Offset + return a.Offset < b.Offset }) - text, marshalErr := m.MarshalToString(respLookupEntry.Entry) if marshalErr != nil { return fmt.Errorf("marshal meta: %v", marshalErr) diff --git a/weed/shell/command_fs_meta_load.go b/weed/shell/command_fs_meta_load.go index 46dc07e9a..7fe4cf809 100644 --- a/weed/shell/command_fs_meta_load.go +++ b/weed/shell/command_fs_meta_load.go @@ -48,7 +48,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io. var dirCount, fileCount uint64 - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { sizeBuf := make([]byte, 4) diff --git a/weed/shell/command_fs_meta_notify.go b/weed/shell/command_fs_meta_notify.go index 4342fa81d..5982214f9 100644 --- a/weed/shell/command_fs_meta_notify.go +++ b/weed/shell/command_fs_meta_notify.go @@ -21,7 +21,7 @@ func (c *commandFsMetaNotify) Name() string { } func (c *commandFsMetaNotify) Help() string { - return `recursively send directory and file meta data to notifiction message queue + return `recursively send directory and file meta data to notification message queue fs.meta.notify # send meta data from current directory to notification message queue diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index b6f2a9172..ea909a527 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -3,6 +3,7 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "io" "os" "path/filepath" @@ -62,8 +63,8 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. fileName := *outputFileName if fileName == "" { t := time.Now() - fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta", - commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + fileName = fmt.Sprintf("%s-%4d%02d%02d-%02d%02d%02d.meta", + commandEnv.option.FilerAddress.ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) } dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) @@ -77,15 +78,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. cipherKey = util.GenCipherKey() } - err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(outputChan chan interface{}) { - sizeBuf := make([]byte, 4) - for item := range outputChan { - b := item.([]byte) - util.Uint32toBytes(sizeBuf, uint32(len(b))) - dst.Write(sizeBuf) - dst.Write(b) - } - }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { + err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if !entry.Entry.IsDirectory { ext := filepath.Ext(entry.Entry.Name) if encrypted, encErr := util.Encrypt([]byte(entry.Entry.Name), cipherKey); encErr == nil { @@ -101,17 +94,25 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. outputChan <- bytes return nil + }, func(outputChan chan interface{}) { + sizeBuf := make([]byte, 4) + for item := range outputChan { + b := item.([]byte) + util.Uint32toBytes(sizeBuf, uint32(len(b))) + dst.Write(sizeBuf) + dst.Write(b) + } }) if err == nil { - fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", commandEnv.option.FilerHost, commandEnv.option.FilerPort, path, fileName) + fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.option.FilerAddress.ToHttpAddress(), path, fileName) } return err } -func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, saveFn func(outputChan chan interface{}), genFn func(entry *filer_pb.FullEntry, outputChan chan interface{}) error) error { +func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, genFn func(entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{})) error { var wg sync.WaitGroup wg.Add(1) @@ -125,6 +126,10 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, err := filer_pb.TraverseBfs(filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { + if strings.HasPrefix(string(parentPath), filer.SystemLogDir) { + return + } + protoMessage := &filer_pb.FullEntry{ Dir: string(parentPath), Entry: entry, diff --git a/weed/shell/command_fs_mkdir.go b/weed/shell/command_fs_mkdir.go index 11b663eec..71a35cb68 100644 --- a/weed/shell/command_fs_mkdir.go +++ b/weed/shell/command_fs_mkdir.go @@ -36,7 +36,7 @@ func (c *commandFsMkdir) Do(args []string, commandEnv *CommandEnv, writer io.Wri dir, name := util.FullPath(path).DirAndName() - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { _, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{ Directory: dir, diff --git a/weed/shell/command_fs_mv.go b/weed/shell/command_fs_mv.go index 2448c8f61..612cdc84e 100644 --- a/weed/shell/command_fs_mv.go +++ b/weed/shell/command_fs_mv.go @@ -54,7 +54,7 @@ func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer destinationDir, destinationName := util.FullPath(destinationPath).DirAndName() - return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // collect destination entry info destinationRequest := &filer_pb.LookupDirectoryEntryRequest{ diff --git a/weed/shell/command_fs_rm.go b/weed/shell/command_fs_rm.go index b383366ca..2ce3275bb 100644 --- a/weed/shell/command_fs_rm.go +++ b/weed/shell/command_fs_rm.go @@ -56,7 +56,7 @@ func (c *commandFsRm) Do(args []string, commandEnv *CommandEnv, writer io.Writer return fmt.Errorf("need to have arguments") } - commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { for _, entry := range entiries { targetPath, err := commandEnv.parseUrl(entry) if err != nil { diff --git a/weed/shell/command_fs_lock_unlock.go b/weed/shell/command_lock_unlock.go similarity index 100% rename from weed/shell/command_fs_lock_unlock.go rename to weed/shell/command_lock_unlock.go diff --git a/weed/shell/command_mount_configure.go b/weed/shell/command_mount_configure.go new file mode 100644 index 000000000..8c268d35c --- /dev/null +++ b/weed/shell/command_mount_configure.go @@ -0,0 +1,64 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/mount_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + _ "google.golang.org/grpc/resolver/passthrough" + "io" +) + +func init() { + Commands = append(Commands, &commandMountConfigure{}) +} + +type commandMountConfigure struct { +} + +func (c *commandMountConfigure) Name() string { + return "mount.configure" +} + +func (c *commandMountConfigure) Help() string { + return `configure the mount on current server + + mount.configure -dir= + + This command connects with local mount via unix socket, so it can only run locally. + The "mount_directory" value needs to be exactly the same as how mount was started in "weed mount -dir=" + +` +} + +func (c *commandMountConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + mountConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + mountDir := mountConfigureCommand.String("dir", "", "the mount directory same as how \"weed mount -dir=\" was started") + mountQuota := mountConfigureCommand.Int("quotaMB", 0, "the quota in MB") + if err = mountConfigureCommand.Parse(args); err != nil { + return nil + } + + mountDirHash := util.HashToInt32([]byte(*mountDir)) + if mountDirHash < 0 { + mountDirHash = -mountDirHash + } + localSocket := fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash) + + clientConn, err := grpc.Dial("passthrough:///unix://"+localSocket, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return + } + defer clientConn.Close() + + client := mount_pb.NewSeaweedMountClient(clientConn) + _, err = client.Configure(context.Background(), &mount_pb.ConfigureRequest{ + CollectionCapacity: int64(*mountQuota) * 1024 * 1024, + }) + + return +} diff --git a/weed/shell/command_remote_cache.go b/weed/shell/command_remote_cache.go index 2888ec979..2bd8d4eda 100644 --- a/weed/shell/command_remote_cache.go +++ b/weed/shell/command_remote_cache.go @@ -26,7 +26,7 @@ func (c *commandRemoteCache) Help() string { return `cache the file content for mounted directories or files # assume a remote storage is configured to name "cloud1" - remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy + remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy # mount and pull one bucket remote.mount -dir=/xxx -remote=cloud1/bucket @@ -50,7 +50,7 @@ func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - dir := remoteMountCommand.String("dir", "", "a directory in filer") + dir := remoteMountCommand.String("dir", "", "a mounted directory or one of its sub folders in filer") concurrency := remoteMountCommand.Int("concurrent", 32, "concurrent file downloading") fileFiler := newFileFilter(remoteMountCommand) @@ -58,15 +58,37 @@ func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io return nil } - mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir) - if detectErr != nil{ + if *dir != "" { + if err := c.doCacheOneDirectory(commandEnv, writer, *dir, fileFiler, *concurrency); err != nil { + return err + } + return nil + } + + mappings, err := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + if err != nil { + return err + } + + for key, _ := range mappings.Mappings { + if err := c.doCacheOneDirectory(commandEnv, writer, key, fileFiler, *concurrency); err != nil { + return err + } + } + + return nil +} + +func (c *commandRemoteCache) doCacheOneDirectory(commandEnv *CommandEnv, writer io.Writer, dir string, fileFiler *FileFilter, concurrency int) error { + mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, dir) + if detectErr != nil { jsonPrintln(writer, mappings) return detectErr } // pull content from remote - if err = c.cacheContentData(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), fileFiler, remoteStorageConf, *concurrency); err != nil { - return fmt.Errorf("cache content data: %v", err) + if err := c.cacheContentData(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(dir), fileFiler, remoteStorageConf, concurrency); err != nil { + return fmt.Errorf("cache content data on %s: %v", localMountedDir, err) } return nil @@ -130,7 +152,7 @@ func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io. return true // true means recursive traversal should continue } - if fileFilter.matches(entry) { + if !fileFilter.matches(entry) { return true } @@ -141,10 +163,10 @@ func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io. remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dir.Child(entry.Name)) - if err := filer.DownloadToLocal(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil { - fmt.Fprintf(writer, "DownloadToLocal %+v: %v\n", remoteLocation, err) + if err := filer.CacheRemoteObjectToLocalCluster(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil { + fmt.Fprintf(writer, "CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err) if executionErr == nil { - executionErr = fmt.Errorf("DownloadToLocal %+v: %v\n", remoteLocation, err) + executionErr = fmt.Errorf("CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err) } return } diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index b6c4af178..e15090190 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -7,6 +7,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" @@ -33,12 +34,14 @@ func (c *commandRemoteConfigure) Help() string { remote.configure # set or update a configuration - remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy - remote.configure -name=cloud2 -type=gcs -gcs.appCredentialsFile=~/service-account-file.json + remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2 + remote.configure -name=cloud2 -type=gcs -gcs.appCredentialsFile=~/service-account-file.json -gcs.projectId=yyy remote.configure -name=cloud3 -type=azure -azure.account_name=xxx -azure.account_key=yyy remote.configure -name=cloud4 -type=aliyun -aliyun.access_key=xxx -aliyun.secret_key=yyy -aliyun.endpoint=oss-cn-shenzhen.aliyuncs.com -aliyun.region=cn-sehnzhen remote.configure -name=cloud5 -type=tencent -tencent.secret_id=xxx -tencent.secret_key=yyy -tencent.endpoint=cos.ap-guangzhou.myqcloud.com remote.configure -name=cloud6 -type=wasabi -wasabi.access_key=xxx -wasabi.secret_key=yyy -wasabi.endpoint=s3.us-west-1.wasabisys.com -wasabi.region=us-west-1 + remote.configure -name=cloud7 -type=storj -storj.access_key=xxx -storj.secret_key=yyy -storj.endpoint=https://gateway.us1.storjshare.io + remote.configure -name=cloud8 -type=filebase -filebase.access_key=xxx -filebase.secret_key=yyy -filebase.endpoint=https://s3.filebase.com # delete one configuration remote.configure -delete -name=cloud1 @@ -58,7 +61,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name") remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage") - remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi|hdfs] storage type") + remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", fmt.Sprintf("[%s] storage type", remote_storage.GetAllRemoteStorageNames())) remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key") remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key") @@ -66,8 +69,10 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.S3Endpoint, "s3.endpoint", "", "endpoint for s3-compatible local object store") remoteConfigureCommand.StringVar(&conf.S3StorageClass, "s3.storage_class", "", "s3 storage class") remoteConfigureCommand.BoolVar(&conf.S3ForcePathStyle, "s3.force_path_style", true, "s3 force path style") + remoteConfigureCommand.BoolVar(&conf.S3V4Signature, "s3.v4_signature", false, "s3 V4 signature") remoteConfigureCommand.StringVar(&conf.GcsGoogleApplicationCredentials, "gcs.appCredentialsFile", "", "google cloud storage credentials file, default to use env GOOGLE_APPLICATION_CREDENTIALS") + remoteConfigureCommand.StringVar(&conf.GcsProjectId, "gcs.projectId", "", "google cloud storage project id, default to use env GOOGLE_CLOUD_PROJECT") remoteConfigureCommand.StringVar(&conf.AzureAccountName, "azure.account_name", "", "azure account name, default to use env AZURE_STORAGE_ACCOUNT") remoteConfigureCommand.StringVar(&conf.AzureAccountKey, "azure.account_key", "", "azure account name, default to use env AZURE_STORAGE_ACCESS_KEY") @@ -95,16 +100,13 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm") remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region") - var namenodes arrayFlags - remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020") - remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name") - remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode - -Example: hdfs/namenode.hadoop.docker -Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'. -`) - remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection") + remoteConfigureCommand.StringVar(&conf.FilebaseAccessKey, "filebase.access_key", "", "Filebase access key") + remoteConfigureCommand.StringVar(&conf.FilebaseSecretKey, "filebase.secret_key", "", "Filebase secret key") + remoteConfigureCommand.StringVar(&conf.FilebaseEndpoint, "filebase.endpoint", "", "Filebase endpoint, https://s3.filebase.com") + remoteConfigureCommand.StringVar(&conf.StorjAccessKey, "storj.access_key", "", "Storj access key") + remoteConfigureCommand.StringVar(&conf.StorjSecretKey, "storj.secret_key", "", "Storj secret key") + remoteConfigureCommand.StringVar(&conf.StorjEndpoint, "storj.endpoint", "", "Storj endpoint") if err = remoteConfigureCommand.Parse(args); err != nil { return nil @@ -148,7 +150,14 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err) } + // change secret key to stars conf.S3SecretKey = strings.Repeat("*", len(conf.S3SecretKey)) + conf.AliyunSecretKey = strings.Repeat("*", len(conf.AliyunSecretKey)) + conf.BaiduAccessKey = strings.Repeat("*", len(conf.BaiduAccessKey)) + conf.FilebaseSecretKey = strings.Repeat("*", len(conf.FilebaseSecretKey)) + conf.StorjSecretKey = strings.Repeat("*", len(conf.StorjSecretKey)) + conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey)) + conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey)) m := jsonpb.Marshaler{ EmitDefaults: false, @@ -165,7 +174,7 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE func (c *commandRemoteConfigure) deleteRemoteStorage(commandEnv *CommandEnv, writer io.Writer, storageName string) error { - return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.DeleteEntryRequest{ Directory: filer.DirectoryEtcRemote, @@ -195,7 +204,7 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write return err } - if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, conf.Name+filer.REMOTE_STORAGE_CONF_SUFFIX, data) }); err != nil && err != filer_pb.ErrNotFound { return err @@ -204,14 +213,3 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write return nil } - -type arrayFlags []string - -func (i *arrayFlags) String() string { - return "my string representation" -} - -func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) - return nil -} diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go index 17b9abdb8..e09a66761 100644 --- a/weed/shell/command_remote_meta_sync.go +++ b/weed/shell/command_remote_meta_sync.go @@ -27,7 +27,7 @@ func (c *commandRemoteMetaSync) Help() string { return `synchronize the local file meta data with the remote file metadata # assume a remote storage is configured to name "cloud1" - remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy + remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy # mount and pull one bucket remote.mount -dir=/xxx -remote=cloud1/bucket @@ -62,7 +62,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer // pull metadata from remote if err = pullMetadata(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf); err != nil { - return fmt.Errorf("cache content data: %v", err) + return fmt.Errorf("cache meta data: %v", err) } return nil @@ -76,7 +76,7 @@ func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*rem This function update entry.RemoteEntry if the remote has any changes. To pull remote updates, or created for the first time, the criteria is: - entry == nil or (entry.RemoteEntry != nil and entry.RemoteEntry.RemoteTag != remote.RemoteTag) + entry == nil or (entry.RemoteEntry != nil and (entry.RemoteEntry.RemoteTag != remote.RemoteTag or entry.RemoteEntry.RemoteMTime < remote.RemoteMTime )) After the meta pull, the entry.RemoteEntry will have: remoteEntry.LastLocalSyncTsNs == 0 Attributes.FileSize = uint64(remoteEntry.RemoteSize) @@ -117,7 +117,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToCache) - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { ctx := context.Background() err = remoteStorage.Traverse(remote, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { localDir := filer.MapRemoteStorageLocationPathToFullPath(localMountedDir, remoteMountedLocation, remoteDir) diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 3e92428d9..019bc93c9 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -12,7 +12,9 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "io" + "os" "strings" + "time" ) func init() { @@ -30,7 +32,7 @@ func (c *commandRemoteMount) Help() string { return `mount remote storage and pull its metadata # assume a remote storage is configured to name "cloud1" - remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy + remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy # mount and pull one bucket remote.mount -dir=/xxx -remote=cloud1/bucket @@ -72,12 +74,12 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io } // sync metadata from remote - if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil { + if err = syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil { return fmt.Errorf("pull metadata: %v", err) } // store a mount configuration in filer - if err = c.saveMountMapping(commandEnv, writer, *dir, remoteStorageLocation); err != nil { + if err = filer.InsertMountMapping(commandEnv, *dir, remoteStorageLocation); err != nil { return fmt.Errorf("save mount mapping: %v", err) } @@ -112,24 +114,33 @@ func jsonPrintln(writer io.Writer, message proto.Message) error { return err } -func (c *commandRemoteMount) findRemoteStorageConfiguration(commandEnv *CommandEnv, writer io.Writer, remote *remote_pb.RemoteStorageLocation) (conf *remote_pb.RemoteConf, err error) { - - return filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote.Name) - -} - -func (c *commandRemoteMount) syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error { +func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error { // find existing directory, and ensure the directory is empty - err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { parent, name := util.FullPath(dir).DirAndName() _, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ Directory: parent, Name: name, }) if lookupErr != nil { - if !strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) { - return fmt.Errorf("lookup %s: %v", dir, lookupErr) + if strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) { + _, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{ + Directory: parent, + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0644 | os.ModeDir), + }, + RemoteEntry: &filer_pb.RemoteEntry{ + StorageName: remoteConf.Name, + }, + }, + }) + return createErr } } @@ -157,38 +168,7 @@ func (c *commandRemoteMount) syncMetadata(commandEnv *CommandEnv, writer io.Writ // pull metadata from remote if err = pullMetadata(commandEnv, writer, util.FullPath(dir), remote, util.FullPath(dir), remoteConf); err != nil { - return fmt.Errorf("cache content data: %v", err) - } - - return nil -} - -func (c *commandRemoteMount) saveMountMapping(commandEnv *CommandEnv, writer io.Writer, dir string, remoteStorageLocation *remote_pb.RemoteStorageLocation) (err error) { - - // read current mapping - var oldContent, newContent []byte - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) - return err - }) - if err != nil { - if err != filer_pb.ErrNotFound { - return fmt.Errorf("read existing mapping: %v", err) - } - } - - // add new mapping - newContent, err = filer.AddRemoteStorageMapping(oldContent, dir, remoteStorageLocation) - if err != nil { - return fmt.Errorf("add mapping %s~%s: %v", dir, remoteStorageLocation, err) - } - - // save back - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent) - }) - if err != nil { - return fmt.Errorf("save mapping: %v", err) + return fmt.Errorf("cache metadata: %v", err) } return nil diff --git a/weed/shell/command_remote_mount_buckets.go b/weed/shell/command_remote_mount_buckets.go new file mode 100644 index 000000000..3ed22936f --- /dev/null +++ b/weed/shell/command_remote_mount_buckets.go @@ -0,0 +1,121 @@ +package shell + +import ( + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" + "github.com/chrislusf/seaweedfs/weed/util" + "io" + "path/filepath" + "regexp" +) + +func init() { + Commands = append(Commands, &commandRemoteMountBuckets{}) +} + +type commandRemoteMountBuckets struct { +} + +func (c *commandRemoteMountBuckets) Name() string { + return "remote.mount.buckets" +} + +func (c *commandRemoteMountBuckets) Help() string { + return `mount all buckets in remote storage and pull its metadata + + # assume a remote storage is configured to name "cloud1" + remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy + + # mount all buckets + remote.mount.buckets -remote=cloud1 + + # after mount, start a separate process to write updates to remote storage + weed filer.remote.sync -filer=: -createBucketAt=cloud1 + +` +} + +func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + remoteMountBucketsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + + remote := remoteMountBucketsCommand.String("remote", "", "an already configured storage name") + bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'") + trimBucketSuffix := remoteMountBucketsCommand.Bool("trimBucketSuffix", true, "remote suffix auto generated by 'weed filer.remote.sync'") + apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets") + if err = remoteMountBucketsCommand.Parse(args); err != nil { + return nil + } + infoAboutSimulationMode(writer, *apply, "-apply") + + if *remote == "" { + _, err = listExistingRemoteStorageMounts(commandEnv, writer) + return err + } + + // find configuration for remote storage + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *remote) + if err != nil { + return fmt.Errorf("find configuration for %s: %v", *remote, err) + } + + // get storage client + remoteStorageClient, err := remote_storage.GetRemoteStorage(remoteConf) + if err != nil { + return fmt.Errorf("get storage client for %s: %v", *remote, err) + } + + buckets, err := remoteStorageClient.ListBuckets() + if err != nil { + return fmt.Errorf("list buckets on %s: %v", *remote, err) + } + + fillerBucketsPath, err := readFilerBucketsPath(commandEnv) + if err != nil { + return fmt.Errorf("read filer buckets path: %v", err) + } + + hasSuffixPattern, _ := regexp.Compile(".+-[0-9][0-9][0-9][0-9]") + + for _, bucket := range buckets { + if *bucketPattern != "" { + if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched { + continue + } + } + + fmt.Fprintf(writer, "bucket %s\n", bucket.Name) + localBucketName := bucket.Name + if *trimBucketSuffix { + if hasSuffixPattern.MatchString(localBucketName) { + localBucketName = localBucketName[:len(localBucketName)-5] + fmt.Fprintf(writer, " mount bucket %s as %s\n", bucket.Name, localBucketName) + } + } + if *apply { + + dir := util.FullPath(fillerBucketsPath).Child(localBucketName) + remoteStorageLocation := &remote_pb.RemoteStorageLocation{ + Name: *remote, + Bucket: bucket.Name, + Path: "/", + } + + // sync metadata from remote + if err = syncMetadata(commandEnv, writer, string(dir), true, remoteConf, remoteStorageLocation); err != nil { + return fmt.Errorf("pull metadata on %+v: %v", remoteStorageLocation, err) + } + + // store a mount configuration in filer + if err = filer.InsertMountMapping(commandEnv, string(dir), remoteStorageLocation); err != nil { + return fmt.Errorf("save mount mapping %s to %+v: %v", dir, remoteStorageLocation, err) + } + + } + } + + return nil +} diff --git a/weed/shell/command_remote_uncache.go b/weed/shell/command_remote_uncache.go index 369f2b3d4..a3433621e 100644 --- a/weed/shell/command_remote_uncache.go +++ b/weed/shell/command_remote_uncache.go @@ -4,12 +4,13 @@ import ( "context" "flag" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/util" "io" "path/filepath" "strings" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" ) func init() { @@ -41,12 +42,12 @@ func (c *commandRemoteUncache) Help() string { func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - remoteUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + remoteUncacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - dir := remoteUnmountCommand.String("dir", "", "a directory in filer") - fileFiler := newFileFilter(remoteUnmountCommand) + dir := remoteUncacheCommand.String("dir", "", "a directory in filer") + fileFiler := newFileFilter(remoteUncacheCommand) - if err = remoteUnmountCommand.Parse(args); err != nil { + if err = remoteUncacheCommand.Parse(args); err != nil { return nil } @@ -54,27 +55,30 @@ func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer if listErr != nil { return listErr } - if *dir == "" { - jsonPrintln(writer, mappings) - fmt.Fprintln(writer, "need to specify '-dir' option") - return nil - } + if *dir != "" { + var localMountedDir string + for k := range mappings.Mappings { + if strings.HasPrefix(*dir, k) { + localMountedDir = k + } + } + if localMountedDir == "" { + jsonPrintln(writer, mappings) + fmt.Fprintf(writer, "%s is not mounted\n", *dir) + return nil + } - var localMountedDir string - for k := range mappings.Mappings { - if strings.HasPrefix(*dir, k) { - localMountedDir = k + // pull content from remote + if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil { + return fmt.Errorf("uncache content data: %v", err) } - } - if localMountedDir == "" { - jsonPrintln(writer, mappings) - fmt.Fprintf(writer, "%s is not mounted\n", *dir) return nil } - // pull content from remote - if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil { - return fmt.Errorf("cache content data: %v", err) + for key, _ := range mappings.Mappings { + if err := c.uncacheContentData(commandEnv, writer, util.FullPath(key), fileFiler); err != nil { + return err + } } return nil @@ -88,7 +92,7 @@ func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer return true // true means recursive traversal should continue } - if fileFilter.matches(entry) { + if !fileFilter.matches(entry) { return true } @@ -101,7 +105,7 @@ func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name)) - err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { _, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{ Directory: string(dir), Entry: entry, @@ -141,12 +145,12 @@ func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) { func (ff *FileFilter) matches(entry *filer_pb.Entry) bool { if *ff.include != "" { if ok, _ := filepath.Match(*ff.include, entry.Name); !ok { - return true + return false } } if *ff.exclude != "" { if ok, _ := filepath.Match(*ff.exclude, entry.Name); ok { - return true + return false } } if *ff.minSize != -1 { @@ -169,5 +173,5 @@ func (ff *FileFilter) matches(entry *filer_pb.Entry) bool { return false } } - return false + return true } diff --git a/weed/shell/command_remote_unmount.go b/weed/shell/command_remote_unmount.go index 9b61f5cfb..49b07f8f1 100644 --- a/weed/shell/command_remote_unmount.go +++ b/weed/shell/command_remote_unmount.go @@ -6,8 +6,10 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" "io" + "time" ) func init() { @@ -25,14 +27,12 @@ func (c *commandRemoteUnmount) Help() string { return `unmount remote storage # assume a remote storage is configured to name "s3_1" - remote.configure -name=s3_1 -type=s3 -access_key=xxx -secret_key=yyy + remote.configure -name=s3_1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy # mount and pull one bucket remote.mount -dir=/xxx -remote=s3_1/bucket # unmount the mounted directory and remove its cache - # Make sure you have stopped "weed filer.remote.sync" first! - # Otherwise, the deletion will also be propagated to the remote storage!!! - remote.unmount -dir=/xxx -iHaveStoppedRemoteSync + remote.unmount -dir=/xxx ` } @@ -42,7 +42,6 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) dir := remoteMountCommand.String("dir", "", "a directory in filer") - hasStoppedRemoteSync := remoteMountCommand.Bool("iHaveStoppedRemoteSync", false, "confirm to stop weed filer.remote.sync first") if err = remoteMountCommand.Parse(args); err != nil { return nil @@ -61,17 +60,21 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer return fmt.Errorf("directory %s is not mounted", *dir) } - if !*hasStoppedRemoteSync { - return fmt.Errorf("make sure \"weed filer.remote.sync\" is stopped to avoid data loss") + // store a mount configuration in filer + fmt.Fprintf(writer, "deleting mount for %s ...\n", *dir) + if err = filer.DeleteMountMapping(commandEnv, *dir); err != nil { + return fmt.Errorf("delete mount mapping: %v", err) } + // purge mounted data + fmt.Fprintf(writer, "purge %s ...\n", *dir) if err = c.purgeMountedData(commandEnv, *dir); err != nil { return fmt.Errorf("purge mounted data: %v", err) } - // store a mount configuration in filer - if err = c.deleteMountMapping(commandEnv, *dir); err != nil { - return fmt.Errorf("delete mount mapping: %v", err) + // reset remote sync offset in case the folder is mounted again + if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *dir, time.Now().UnixNano()); err != nil { + return fmt.Errorf("reset remote.sync offset for %s: %v", *dir, err) } return nil @@ -80,7 +83,7 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir string) error { // find existing directory, and ensure the directory is empty - err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { parent, name := util.FullPath(dir).DirAndName() lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ Directory: parent, @@ -100,6 +103,8 @@ func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir stri mkdirErr := filer_pb.DoMkdir(client, parent, name, func(entry *filer_pb.Entry) { entry.Attributes = oldEntry.Attributes entry.Extended = oldEntry.Extended + entry.Attributes.Crtime = time.Now().Unix() + entry.Attributes.Mtime = time.Now().Unix() }) if mkdirErr != nil { return fmt.Errorf("mkdir %s: %v", dir, mkdirErr) @@ -113,34 +118,3 @@ func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir stri return nil } - -func (c *commandRemoteUnmount) deleteMountMapping(commandEnv *CommandEnv, dir string) (err error) { - - // read current mapping - var oldContent, newContent []byte - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - oldContent, err = filer.ReadInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) - return err - }) - if err != nil { - if err != filer_pb.ErrNotFound { - return fmt.Errorf("read existing mapping: %v", err) - } - } - - // add new mapping - newContent, err = filer.RemoveRemoteStorageMapping(oldContent, dir) - if err != nil { - return fmt.Errorf("delete mount %s: %v", dir, err) - } - - // save back - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE, newContent) - }) - if err != nil { - return fmt.Errorf("save mapping: %v", err) - } - - return nil -} diff --git a/weed/shell/command_s3_bucket_create.go b/weed/shell/command_s3_bucket_create.go index a512ffc4a..ffee40be5 100644 --- a/weed/shell/command_s3_bucket_create.go +++ b/weed/shell/command_s3_bucket_create.go @@ -26,7 +26,7 @@ func (c *commandS3BucketCreate) Help() string { return `create a bucket with a given name Example: - s3.bucket.create -name -replication 001 + s3.bucket.create -name ` } @@ -34,9 +34,6 @@ func (c *commandS3BucketCreate) Do(args []string, commandEnv *CommandEnv, writer bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) bucketName := bucketCommand.String("name", "", "bucket name") - replication := bucketCommand.String("replication", "", "replication setting for the bucket, if not set "+ - "it will honor the value defined by the filer if it exists, "+ - "else it will honor the value defined on the master") if err = bucketCommand.Parse(args); err != nil { return nil } @@ -45,7 +42,7 @@ func (c *commandS3BucketCreate) Do(args []string, commandEnv *CommandEnv, writer return fmt.Errorf("empty bucket name") } - err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { @@ -59,11 +56,9 @@ func (c *commandS3BucketCreate) Do(args []string, commandEnv *CommandEnv, writer Name: *bucketName, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(0777 | os.ModeDir), - Collection: *bucketName, - Replication: *replication, + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0777 | os.ModeDir), }, } diff --git a/weed/shell/command_s3_bucket_delete.go b/weed/shell/command_s3_bucket_delete.go index 26953c249..5f8585b0b 100644 --- a/weed/shell/command_s3_bucket_delete.go +++ b/weed/shell/command_s3_bucket_delete.go @@ -52,7 +52,7 @@ func (c *commandS3BucketDelete) Do(args []string, commandEnv *CommandEnv, writer } // delete the collection directly first - err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err = client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ Name: *bucketName, }) diff --git a/weed/shell/command_s3_bucket_list.go b/weed/shell/command_s3_bucket_list.go index 0c4e8d18f..09bde08b9 100644 --- a/weed/shell/command_s3_bucket_list.go +++ b/weed/shell/command_s3_bucket_list.go @@ -34,6 +34,14 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i return nil } + // collect collection information + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) + if err != nil { + return err + } + collectionInfos := make(map[string]*CollectionInfo) + collectCollectionInfo(topologyInfo, collectionInfos) + _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) if parseErr != nil { return parseErr @@ -49,11 +57,17 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i if !entry.IsDirectory { return nil } - if entry.Attributes.Replication == "" || entry.Attributes.Replication == "000" { - fmt.Fprintf(writer, " %s\n", entry.Name) - } else { - fmt.Fprintf(writer, " %s\t\t\treplication: %s\n", entry.Name, entry.Attributes.Replication) + collection := entry.Name + var collectionSize, fileCount float64 + if collectionInfo, found := collectionInfos[collection]; found { + collectionSize = collectionInfo.Size + fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount + } + fmt.Fprintf(writer, " %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount) + if entry.Quota > 0 { + fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota)) } + fmt.Fprintln(writer) return nil }, "", false, math.MaxUint32) if err != nil { @@ -65,7 +79,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i } func readFilerBucketsPath(filerClient filer_pb.FilerClient) (filerBucketsPath string, err error) { - err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { diff --git a/weed/shell/command_s3_bucket_quota.go b/weed/shell/command_s3_bucket_quota.go new file mode 100644 index 000000000..17be1d5b0 --- /dev/null +++ b/weed/shell/command_s3_bucket_quota.go @@ -0,0 +1,93 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandS3BucketQuota{}) +} + +type commandS3BucketQuota struct { +} + +func (c *commandS3BucketQuota) Name() string { + return "s3.bucket.quota" +} + +func (c *commandS3BucketQuota) Help() string { + return `set/remove/enable/disable quota for a bucket + + Example: + s3.bucket.quota -name= -operation=set -sizeMB=1024 +` +} + +func (c *commandS3BucketQuota) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + bucketName := bucketCommand.String("name", "", "bucket name") + operationName := bucketCommand.String("op", "set", "operation name [set|remove|enable|disable]") + sizeMB := bucketCommand.Int64("sizeMB", 0, "bucket quota size in MiB") + if err = bucketCommand.Parse(args); err != nil { + return nil + } + + if *bucketName == "" { + return fmt.Errorf("empty bucket name") + } + + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + ctx := context.Background() + + resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer configuration: %v", err) + } + filerBucketsPath := resp.DirBuckets + + lookupResp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{ + Directory: filerBucketsPath, + Name: *bucketName, + }) + if err != nil { + return fmt.Errorf("did not find bucket %s: %v", *bucketName, err) + } + bucketEntry := lookupResp.Entry + + switch *operationName { + case "set": + bucketEntry.Quota = *sizeMB * 1024 * 1024 + case "remove": + bucketEntry.Quota = 0 + case "enable": + if bucketEntry.Quota < 0 { + bucketEntry.Quota = -bucketEntry.Quota + } + case "disable": + if bucketEntry.Quota > 0 { + bucketEntry.Quota = -bucketEntry.Quota + } + } + + if err := filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{ + Directory: filerBucketsPath, + Entry: bucketEntry, + }); err != nil { + return err + } + + println("updated quota for bucket", *bucketName) + + return nil + + }) + + return err + +} diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go new file mode 100644 index 000000000..f2a0d14f9 --- /dev/null +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -0,0 +1,137 @@ +package shell + +import ( + "bytes" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" + "math" +) + +func init() { + Commands = append(Commands, &commandS3BucketQuotaEnforce{}) +} + +type commandS3BucketQuotaEnforce struct { +} + +func (c *commandS3BucketQuotaEnforce) Name() string { + return "s3.bucket.quota.enforce" +} + +func (c *commandS3BucketQuotaEnforce) Help() string { + return `check quota for all buckets, make the bucket read only if over the limit + + Example: + s3.bucket.quota.enforce -apply +` +} + +func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + applyQuotaLimit := bucketCommand.Bool("apply", false, "actually change the buckets readonly attribute") + if err = bucketCommand.Parse(args); err != nil { + return nil + } + infoAboutSimulationMode(writer, *applyQuotaLimit, "-apply") + + // collect collection information + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) + if err != nil { + return err + } + collectionInfos := make(map[string]*CollectionInfo) + collectCollectionInfo(topologyInfo, collectionInfos) + + // read buckets path + var filerBucketsPath string + filerBucketsPath, err = readFilerBucketsPath(commandEnv) + if err != nil { + return fmt.Errorf("read buckets: %v", err) + } + + // read existing filer configuration + fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) + if err != nil { + return err + } + + // process each bucket + hasConfChanges := false + err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error { + if !entry.IsDirectory { + return nil + } + collection := entry.Name + var collectionSize float64 + if collectionInfo, found := collectionInfos[collection]; found { + collectionSize = collectionInfo.Size + } + if c.processEachBucket(fc, filerBucketsPath, entry, writer, collectionSize) { + hasConfChanges = true + } + return nil + }, "", false, math.MaxUint32) + if err != nil { + return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err) + } + + // apply the configuration changes + if hasConfChanges && *applyQuotaLimit { + + var buf2 bytes.Buffer + fc.ToText(&buf2) + + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.SaveInsideFiler(client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes()) + }); err != nil && err != filer_pb.ErrNotFound { + return err + } + } + + return err + +} + +func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize float64) (hasConfChanges bool) { + + locPrefix := filerBucketsPath + "/" + entry.Name + "/" + locConf := fc.MatchStorageRule(locPrefix) + locConf.LocationPrefix = locPrefix + + if entry.Quota > 0 { + if locConf.ReadOnly { + if collectionSize < float64(entry.Quota) { + locConf.ReadOnly = false + hasConfChanges = true + } + } else { + if collectionSize > float64(entry.Quota) { + locConf.ReadOnly = true + hasConfChanges = true + } + } + } else { + if locConf.ReadOnly { + locConf.ReadOnly = false + hasConfChanges = true + } + } + + if hasConfChanges { + fmt.Fprintf(writer, " %s\tsize:%.0f", entry.Name, collectionSize) + fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, collectionSize*100/float64(entry.Quota)) + fmt.Fprintln(writer) + if locConf.ReadOnly { + fmt.Fprintf(writer, " changing bucket %s to read only!\n", entry.Name) + } else { + fmt.Fprintf(writer, " changing bucket %s to writable.\n", entry.Name) + } + fc.AddLocationConf(locConf) + } + + return +} diff --git a/weed/shell/command_s3_circuitbreaker.go b/weed/shell/command_s3_circuitbreaker.go new file mode 100644 index 000000000..7e11153bf --- /dev/null +++ b/weed/shell/command_s3_circuitbreaker.go @@ -0,0 +1,358 @@ +package shell + +import ( + "bytes" + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/s3_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "io" + "strconv" + "strings" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +var LoadConfig = loadConfig + +func init() { + Commands = append(Commands, &commandS3CircuitBreaker{}) +} + +type commandS3CircuitBreaker struct { +} + +func (c *commandS3CircuitBreaker) Name() string { + return "s3.circuitBreaker" +} + +func (c *commandS3CircuitBreaker) Help() string { + return `configure and apply s3 circuit breaker options for each bucket + + # examples + # add circuit breaker config for global + s3.circuitBreaker -global -type count -actions Read,Write -values 500,200 -apply + + # disable global config + s3.circuitBreaker -global -disable -apply + + # add circuit breaker config for buckets x,y,z + s3.circuitBreaker -buckets x,y,z -type count -actions Read,Write -values 200,100 -apply + + # disable circuit breaker config of x + s3.circuitBreaker -buckets x -disable -apply + + # delete circuit breaker config of x + s3.circuitBreaker -buckets x -delete -apply + + # clear all circuit breaker config + s3.circuitBreaker -delete -apply + ` +} + +func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + dir := s3_constants.CircuitBreakerConfigDir + file := s3_constants.CircuitBreakerConfigFile + + s3CircuitBreakerCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + buckets := s3CircuitBreakerCommand.String("buckets", "", "the bucket name(s) to configure, eg: -buckets x,y,z") + global := s3CircuitBreakerCommand.Bool("global", false, "configure global circuit breaker") + + actions := s3CircuitBreakerCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin") + limitType := s3CircuitBreakerCommand.String("type", "", "'Count' or 'MB'; Count represents the number of simultaneous requests, and MB represents the content size of all simultaneous requests") + values := s3CircuitBreakerCommand.String("values", "", "comma separated values") + + disabled := s3CircuitBreakerCommand.Bool("disable", false, "disable global or buckets circuit breaker") + deleted := s3CircuitBreakerCommand.Bool("delete", false, "delete circuit breaker config") + + apply := s3CircuitBreakerCommand.Bool("apply", false, "update and apply current configuration") + + if err = s3CircuitBreakerCommand.Parse(args); err != nil { + return nil + + } + + var buf bytes.Buffer + err = LoadConfig(commandEnv, dir, file, &buf) + if err != nil { + return err + } + + cbCfg := &s3_pb.S3CircuitBreakerConfig{ + Buckets: make(map[string]*s3_pb.S3CircuitBreakerOptions), + } + if buf.Len() > 0 { + if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), cbCfg); err != nil { + return err + } + } + + if *deleted { + cmdBuckets, cmdActions, _, err := c.initActionsAndValues(buckets, actions, limitType, values, true) + if err != nil { + return err + } + + if len(cmdBuckets) <= 0 && !*global { + if len(cmdActions) > 0 { + deleteGlobalActions(cbCfg, cmdActions, limitType) + if cbCfg.Buckets != nil { + var allBuckets []string + for bucket := range cbCfg.Buckets { + allBuckets = append(allBuckets, bucket) + } + deleteBucketsActions(allBuckets, cbCfg, cmdActions, limitType) + } + } else { + cbCfg.Global = nil + cbCfg.Buckets = nil + } + } else { + if len(cmdBuckets) > 0 { + deleteBucketsActions(cmdBuckets, cbCfg, cmdActions, limitType) + } + if *global { + deleteGlobalActions(cbCfg, cmdActions, nil) + } + } + } else { + cmdBuckets, cmdActions, cmdValues, err := c.initActionsAndValues(buckets, actions, limitType, values, *disabled) + if err != nil { + return err + } + + if len(cmdActions) > 0 && len(*buckets) <= 0 && !*global { + return fmt.Errorf("one of -global and -buckets must be specified") + } + + if len(*buckets) > 0 { + for _, bucket := range cmdBuckets { + var cbOptions *s3_pb.S3CircuitBreakerOptions + var exists bool + if cbOptions, exists = cbCfg.Buckets[bucket]; !exists { + cbOptions = &s3_pb.S3CircuitBreakerOptions{} + cbCfg.Buckets[bucket] = cbOptions + } + cbOptions.Enabled = !*disabled + + if len(cmdActions) > 0 { + err = insertOrUpdateValues(cbOptions, cmdActions, cmdValues, limitType) + if err != nil { + return err + } + } + + if len(cbOptions.Actions) <= 0 && !cbOptions.Enabled { + delete(cbCfg.Buckets, bucket) + } + } + } + + if *global { + globalOptions := cbCfg.Global + if globalOptions == nil { + globalOptions = &s3_pb.S3CircuitBreakerOptions{Actions: make(map[string]int64, len(cmdActions))} + cbCfg.Global = globalOptions + } + globalOptions.Enabled = !*disabled + + if len(cmdActions) > 0 { + err = insertOrUpdateValues(globalOptions, cmdActions, cmdValues, limitType) + if err != nil { + return err + } + } + + if len(globalOptions.Actions) <= 0 && !globalOptions.Enabled { + cbCfg.Global = nil + } + } + } + + buf.Reset() + err = filer.ProtoToText(&buf, cbCfg) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(writer, string(buf.Bytes())) + _, _ = fmt.Fprintln(writer) + + if *apply { + if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.SaveInsideFiler(client, dir, file, buf.Bytes()) + }); err != nil { + return err + } + } + + return nil +} + +func loadConfig(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error { + if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.ReadEntry(commandEnv.MasterClient, client, dir, file, buf) + }); err != nil && err != filer_pb.ErrNotFound { + return err + } + return nil +} + +func insertOrUpdateValues(cbOptions *s3_pb.S3CircuitBreakerOptions, cmdActions []string, cmdValues []int64, limitType *string) error { + if len(*limitType) == 0 { + return fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed") + } + + if cbOptions.Actions == nil { + cbOptions.Actions = make(map[string]int64, len(cmdActions)) + } + + if len(cmdValues) > 0 { + for i, action := range cmdActions { + cbOptions.Actions[s3_constants.Concat(action, *limitType)] = cmdValues[i] + } + } + return nil +} + +func deleteBucketsActions(cmdBuckets []string, cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) { + if cbCfg.Buckets == nil { + return + } + + if len(cmdActions) == 0 { + for _, bucket := range cmdBuckets { + delete(cbCfg.Buckets, bucket) + } + } else { + for _, bucket := range cmdBuckets { + if cbOption, ok := cbCfg.Buckets[bucket]; ok { + if len(cmdActions) > 0 && cbOption.Actions != nil { + for _, action := range cmdActions { + delete(cbOption.Actions, s3_constants.Concat(action, *limitType)) + } + } + + if len(cbOption.Actions) == 0 && !cbOption.Enabled { + delete(cbCfg.Buckets, bucket) + } + } + } + } + + if len(cbCfg.Buckets) == 0 { + cbCfg.Buckets = nil + } +} + +func deleteGlobalActions(cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) { + globalOptions := cbCfg.Global + if globalOptions == nil { + return + } + + if len(cmdActions) == 0 && globalOptions.Actions != nil { + globalOptions.Actions = nil + return + } else { + for _, action := range cmdActions { + delete(globalOptions.Actions, s3_constants.Concat(action, *limitType)) + } + } + + if len(globalOptions.Actions) == 0 && !globalOptions.Enabled { + cbCfg.Global = nil + } +} + +func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitType, values *string, parseValues bool) (cmdBuckets, cmdActions []string, cmdValues []int64, err error) { + if len(*buckets) > 0 { + cmdBuckets = strings.Split(*buckets, ",") + } + + if len(*actions) > 0 { + cmdActions = strings.Split(*actions, ",") + + //check action valid + for _, action := range cmdActions { + var found bool + for _, allowedAction := range s3_constants.AllowedActions { + if allowedAction == action { + found = true + } + } + if !found { + return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_constants.AllowedActions) + } + } + } + + if !parseValues { + if len(cmdActions) < 0 { + for _, action := range s3_constants.AllowedActions { + cmdActions = append(cmdActions, action) + } + } + + if len(*limitType) > 0 { + switch *limitType { + case s3_constants.LimitTypeCount: + elements := strings.Split(*values, ",") + if len(cmdActions) != len(elements) { + if len(elements) != 1 || len(elements) == 0 { + return nil, nil, nil, fmt.Errorf("count of flag[-actions] and flag[-counts] not equal") + } + v, err := strconv.Atoi(elements[0]) + if err != nil { + return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)") + } + for range cmdActions { + cmdValues = append(cmdValues, int64(v)) + } + } else { + for _, value := range elements { + v, err := strconv.Atoi(value) + if err != nil { + return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)") + } + cmdValues = append(cmdValues, int64(v)) + } + } + case s3_constants.LimitTypeBytes: + elements := strings.Split(*values, ",") + if len(cmdActions) != len(elements) { + if len(elements) != 1 || len(elements) == 0 { + return nil, nil, nil, fmt.Errorf("values count of -actions and -values not equal") + } + v, err := parseMBToBytes(elements[0]) + if err != nil { + return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)") + } + for range cmdActions { + cmdValues = append(cmdValues, v) + } + } else { + for _, value := range elements { + v, err := parseMBToBytes(value) + if err != nil { + return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)") + } + cmdValues = append(cmdValues, v) + } + } + default: + return nil, nil, nil, fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed") + } + } else { + *limitType = "" + } + } + return cmdBuckets, cmdActions, cmdValues, nil +} + +func parseMBToBytes(valStr string) (int64, error) { + v, err := strconv.Atoi(valStr) + v *= 1024 * 1024 + return int64(v), err +} diff --git a/weed/shell/command_s3_circuitbreaker_test.go b/weed/shell/command_s3_circuitbreaker_test.go new file mode 100644 index 000000000..3d0b4ac6e --- /dev/null +++ b/weed/shell/command_s3_circuitbreaker_test.go @@ -0,0 +1,292 @@ +package shell + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + "testing" +) + +type Case struct { + args []string + result string +} + +var ( + TestCases = []*Case{ + //add circuit breaker config for global + { + args: strings.Split("-global -type Count -actions Read,Write -values 500,200", " "), + result: `{ + "global": { + "enabled": true, + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + } + }`, + }, + + //disable global config + { + args: strings.Split("-global -disable", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + } + }`, + }, + + //add circuit breaker config for buckets x,y,z + { + args: strings.Split("-buckets x,y,z -type Count -actions Read,Write -values 200,100", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "x": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //disable circuit breaker config of x + { + args: strings.Split("-buckets x -disable", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "x": { + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //delete circuit breaker config of x + { + args: strings.Split("-buckets x -delete", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //configure the circuit breaker for the size of the uploaded file for bucket x,y + { + args: strings.Split("-buckets x,y -type MB -actions Write -values 1024", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "x": { + "enabled": true, + "actions": { + "Write:MB": "1073741824" + } + }, + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100", + "Write:MB": "1073741824" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //delete the circuit breaker configuration for the size of the uploaded file of bucket x,y + { + args: strings.Split("-buckets x,y -type MB -actions Write -delete", " "), + result: `{ + "global": { + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "x": { + "enabled": true + }, + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //enable global circuit breaker config (without -disable flag) + { + args: strings.Split("-global", " "), + result: `{ + "global": { + "enabled": true, + "actions": { + "Read:Count": "500", + "Write:Count": "200" + } + }, + "buckets": { + "x": { + "enabled": true + }, + "y": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + }, + "z": { + "enabled": true, + "actions": { + "Read:Count": "200", + "Write:Count": "100" + } + } + } + }`, + }, + + //clear all circuit breaker config + { + args: strings.Split("-delete", " "), + result: `{ + + }`, + }, + } +) + +func TestCircuitBreakerShell(t *testing.T) { + var writeBuf bytes.Buffer + cmd := &commandS3CircuitBreaker{} + LoadConfig = func(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error { + _, err := buf.Write(writeBuf.Bytes()) + if err != nil { + return err + } + writeBuf.Reset() + return nil + } + + for i, tc := range TestCases { + err := cmd.Do(tc.args, nil, &writeBuf) + if err != nil { + t.Fatal(err) + } + if i != 0 { + result := writeBuf.String() + + actual := make(map[string]interface{}) + err := json.Unmarshal([]byte(result), &actual) + if err != nil { + t.Error(err) + } + + expect := make(map[string]interface{}) + err = json.Unmarshal([]byte(result), &expect) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(actual, expect) { + t.Fatal("result of s3 circuit breaker shell command is unexpect!") + } + } + } +} diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index 1ba31292c..a6dc8f574 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -3,6 +3,7 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" "io" "math" @@ -39,6 +40,8 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer return nil } + signingKey := util.GetViper().GetString("jwt.signing.key") + var filerBucketsPath string filerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { @@ -55,14 +58,16 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer } for _, bucket := range buckets { - c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo) + if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo, signingKey); err != nil { + fmt.Fprintf(writer, fmt.Sprintf("failed cleanup uploads for backet %s: %v", bucket, err)) + } } return err } -func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error { +func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration, signingKey string) error { uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads" var staleUploads []string now := time.Now() @@ -77,12 +82,17 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io return fmt.Errorf("list uploads under %v: %v", uploadsDir, err) } + var encodedJwt security.EncodedJwt + if signingKey != "" { + encodedJwt = security.GenJwtForFilerServer(security.SigningKey(signingKey), 15*60) + } + for _, staleUpload := range staleUploads { - deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerHost, commandEnv.option.FilerPort, uploadsDir, staleUpload) + deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload) fmt.Fprintf(writer, "purge %s\n", deleteUrl) - err = util.Delete(deleteUrl, "") - if err != nil { + err = util.Delete(deleteUrl, string(encodedJwt)) + if err != nil && err.Error() != "" { return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) } } diff --git a/weed/shell/command_s3_configure.go b/weed/shell/command_s3_configure.go index 5eab2ebd0..ddcafd847 100644 --- a/weed/shell/command_s3_configure.go +++ b/weed/shell/command_s3_configure.go @@ -42,13 +42,12 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io secretKey := s3ConfigureCommand.String("secret_key", "", "specify the secret key") isDelete := s3ConfigureCommand.Bool("delete", false, "delete users, actions or access keys") apply := s3ConfigureCommand.Bool("apply", false, "update and apply s3 configuration") - if err = s3ConfigureCommand.Parse(args); err != nil { return nil } var buf bytes.Buffer - if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return filer.ReadEntry(commandEnv.MasterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf) }); err != nil && err != filer_pb.ErrNotFound { return err @@ -83,6 +82,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io } } if changed { + infoAboutSimulationMode(writer, *apply, "-apply") if *isDelete { var exists []int for _, cmdAction := range cmdActions { @@ -151,6 +151,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io } } } else if *user != "" && *actions != "" { + infoAboutSimulationMode(writer, *apply, "-apply") identity := iam_pb.Identity{ Name: *user, Actions: cmdActions, @@ -171,7 +172,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io if *apply { - if err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()) }); err != nil { return err diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index df3067722..462143e25 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -3,11 +3,12 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "os" - "sort" "time" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -62,10 +63,6 @@ func (c *commandVolumeBalance) Help() string { func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection") dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter") @@ -73,9 +70,14 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer if err = balanceCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyBalancing, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second) if err != nil { return err } @@ -223,14 +225,14 @@ func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool } func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) { - sort.Slice(volumes, func(i, j int) bool { - return volumes[i].Size < volumes[j].Size + slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Size < b.Size }) } func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) { - sort.Slice(volumes, func(i, j int) bool { - return volumes[i].Id < volumes[j].Id + slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Id < b.Id }) } @@ -254,10 +256,9 @@ func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volu for hasMoved { hasMoved = false - sort.Slice(nodesWithCapacity, func(i, j int) bool { - return nodesWithCapacity[i].localVolumeRatio(capacityFunc) < nodesWithCapacity[j].localVolumeRatio(capacityFunc) + slices.SortFunc(nodesWithCapacity, func(a, b *Node) bool { + return a.localVolumeRatio(capacityFunc) < b.localVolumeRatio(capacityFunc) }) - fullNode := nodesWithCapacity[len(nodesWithCapacity)-1] var candidateVolumes []*master_pb.VolumeInformationMessage for _, v := range fullNode.selectedVolumes { @@ -325,7 +326,7 @@ func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, f } fmt.Fprintf(os.Stdout, " moving %s volume %s%d %s => %s\n", v.DiskType, collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id) if applyChange { - return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), fullNode.info.Id, emptyNode.info.Id, 5*time.Second, v.DiskType, false) + return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(fullNode.info), pb.NewServerAddressFromDataNode(emptyNode.info), 5*time.Second, v.DiskType, false) } return nil } diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index 7e060f3d3..6a22a0273 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -6,11 +6,12 @@ import ( "flag" "fmt" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" + "golang.org/x/exp/slices" "io" "math" - "sort" ) func init() { @@ -26,7 +27,7 @@ func (c *commandVolumeCheckDisk) Name() string { } func (c *commandVolumeCheckDisk) Help() string { - return `check all replicated volumes to find and fix inconsistencies + return `check all replicated volumes to find and fix inconsistencies. It is optional and resource intensive. How it works: @@ -41,23 +42,25 @@ func (c *commandVolumeCheckDisk) Help() string { func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) slowMode := fsckCommand.Bool("slow", false, "slow mode checks all replicas even file counts are the same") verbose := fsckCommand.Bool("v", false, "verbose mode") + volumeId := fsckCommand.Uint("volumeId", 0, "the volume id") applyChanges := fsckCommand.Bool("force", false, "apply the fix") nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit") if err = fsckCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyChanges, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } c.env = commandEnv // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -67,13 +70,13 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write fileCount := func(replica *VolumeReplica) uint64 { return replica.info.FileCount - replica.info.DeleteCount } - aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb() - defer aDB.Close() - defer bDB.Close() for _, replicas := range volumeReplicas { - sort.Slice(replicas, func(i, j int) bool { - return fileCount(replicas[i]) > fileCount(replicas[j]) + if *volumeId > 0 && replicas[0].info.Id != uint32(*volumeId) { + continue + } + slices.SortFunc(replicas, func(a, b *VolumeReplica) bool { + return fileCount(a) > fileCount(b) }) for len(replicas) >= 2 { a, b := replicas[0], replicas[1] @@ -89,7 +92,7 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write continue } - if err := c.syncTwoReplicas(aDB, bDB, a, verbose, writer, b, err, applyChanges, nonRepairThreshold); err != nil { + if err := c.syncTwoReplicas(a, b, *applyChanges, *nonRepairThreshold, *verbose, writer); err != nil { fmt.Fprintf(writer, "sync volume %d on %s and %s: %v\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id, err) } replicas = replicas[1:] @@ -99,42 +102,50 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write return nil } -func (c *commandVolumeCheckDisk) syncTwoReplicas(aDB *needle_map.MemDb, bDB *needle_map.MemDb, a *VolumeReplica, verbose *bool, writer io.Writer, b *VolumeReplica, err error, applyChanges *bool, nonRepairThreshold *float64) error { +func (c *commandVolumeCheckDisk) syncTwoReplicas(a *VolumeReplica, b *VolumeReplica, applyChanges bool, nonRepairThreshold float64, verbose bool, writer io.Writer) (err error) { aHasChanges, bHasChanges := true, true for aHasChanges || bHasChanges { - // reset index db + if aHasChanges, bHasChanges, err = c.checkBoth(a, b, applyChanges, nonRepairThreshold, verbose, writer); err != nil { + return err + } + } + return nil +} + +func (c *commandVolumeCheckDisk) checkBoth(a *VolumeReplica, b *VolumeReplica, applyChanges bool, nonRepairThreshold float64, verbose bool, writer io.Writer) (aHasChanges bool, bHasChanges bool, err error) { + aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb() + defer func() { aDB.Close() bDB.Close() - aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb() + }() - // read index db - if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil { - return err - } - if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil { - return err - } + // read index db + if err = c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, pb.NewServerAddressFromDataNode(a.location.dataNode), verbose, writer); err != nil { + return true, true, fmt.Errorf("readIndexDatabase %s volume %d: %v", a.location.dataNode, a.info.Id, err) + } + if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, pb.NewServerAddressFromDataNode(b.location.dataNode), verbose, writer); err != nil { + return true, true, fmt.Errorf("readIndexDatabase %s volume %d: %v", b.location.dataNode, b.info.Id, err) + } - // find and make up the differences - if aHasChanges, err = c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil { - return err - } - if bHasChanges, err = c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil { - return err - } + // find and make up the differences + if aHasChanges, err = c.doVolumeCheckDisk(bDB, aDB, b, a, verbose, writer, applyChanges, nonRepairThreshold); err != nil { + return true, true, fmt.Errorf("doVolumeCheckDisk source:%s target:%s volume %d: %v", b.location.dataNode, a.location.dataNode, b.info.Id, err) } - return nil + if bHasChanges, err = c.doVolumeCheckDisk(aDB, bDB, a, b, verbose, writer, applyChanges, nonRepairThreshold); err != nil { + return true, true, fmt.Errorf("doVolumeCheckDisk source:%s target:%s volume %d: %v", a.location.dataNode, b.location.dataNode, a.info.Id, err) + } + return } -func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) (hasChanges bool, err error) { +func (c *commandVolumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) (hasChanges bool, err error) { // find missing keys // hash join, can be more efficient var missingNeedles []needle_map.NeedleValue var counter int - subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error { + minuend.AscendingVisit(func(value needle_map.NeedleValue) error { counter++ - if _, found := minuend.Get(value.Key); !found { + if _, found := subtrahend.Get(value.Key); !found { missingNeedles = append(missingNeedles, value) } return nil @@ -155,7 +166,7 @@ func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_m for _, needleValue := range missingNeedles { - needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue) + needleBlob, err := c.readSourceNeedleBlob(pb.NewServerAddressFromDataNode(source.location.dataNode), source.info.Id, needleValue) if err != nil { return hasChanges, err } @@ -170,7 +181,7 @@ func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_m hasChanges = true - if err = c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil { + if err = c.writeNeedleBlobToTarget(pb.NewServerAddressFromDataNode(target.location.dataNode), source.info.Id, needleValue, needleBlob); err != nil { return hasChanges, err } @@ -179,9 +190,9 @@ func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_m return } -func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) { +func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer pb.ServerAddress, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) { - err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{ VolumeId: volumeId, NeedleId: uint64(needleValue.Key), @@ -197,9 +208,9 @@ func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, return } -func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error { +func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer pb.ServerAddress, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error { - return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { _, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{ VolumeId: volumeId, NeedleId: uint64(needleValue.Key), @@ -211,7 +222,7 @@ func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer stri } -func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error { +func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer pb.ServerAddress, verbose bool, writer io.Writer) error { var buf bytes.Buffer if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil { @@ -226,9 +237,9 @@ func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collect } -func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error { +func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer pb.ServerAddress, buf *bytes.Buffer, verbose bool, writer io.Writer) error { - return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(true, volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { ext := ".idx" diff --git a/weed/shell/command_volume_configure_replication.go b/weed/shell/command_volume_configure_replication.go index e3f034873..610986489 100644 --- a/weed/shell/command_volume_configure_replication.go +++ b/weed/shell/command_volume_configure_replication.go @@ -5,7 +5,9 @@ import ( "errors" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" + "path/filepath" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -33,19 +35,20 @@ func (c *commandVolumeConfigureReplication) Help() string { ` } -func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - - if err = commandEnv.confirmIsLocked(); err != nil { - return - } +func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *CommandEnv, _ io.Writer) (err error) { configureReplicationCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := configureReplicationCommand.Int("volumeId", 0, "the volume id") replicationString := configureReplicationCommand.String("replication", "", "the intended replication value") + collectionPattern := configureReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") if err = configureReplicationCommand.Parse(args); err != nil { return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + if *replicationString == "" { return fmt.Errorf("empty replication value") } @@ -54,15 +57,15 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman if err != nil { return fmt.Errorf("replication format: %v", err) } - replicaPlacementInt32 := uint32(replicaPlacement.Byte()) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } vid := needle.VolumeId(*volumeIdInt) + volumeFilter := getVolumeFilter(replicaPlacement, uint32(vid), *collectionPattern) // find all data nodes with volumes that needs replication change var allLocations []location @@ -70,7 +73,7 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman loc := newLocation(dc, string(rack), dn) for _, diskInfo := range dn.DiskInfos { for _, v := range diskInfo.VolumeInfos { - if v.Id == uint32(vid) && v.ReplicaPlacement != replicaPlacementInt32 { + if volumeFilter(v) { allLocations = append(allLocations, loc) continue } @@ -83,7 +86,7 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman } for _, dst := range allLocations { - err := operation.WithVolumeServerClient(dst.dataNode.Id, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, configureErr := volumeServerClient.VolumeConfigure(context.Background(), &volume_server_pb.VolumeConfigureRequest{ VolumeId: uint32(vid), Replication: replicaPlacement.String(), @@ -105,3 +108,19 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman return nil } + +func getVolumeFilter(replicaPlacement *super_block.ReplicaPlacement, volumeId uint32, collectionPattern string) func(message *master_pb.VolumeInformationMessage) bool { + replicaPlacementInt32 := uint32(replicaPlacement.Byte()) + if volumeId > 0 { + return func(v *master_pb.VolumeInformationMessage) bool { + return v.Id == volumeId && v.ReplicaPlacement != replicaPlacementInt32 + } + } + return func(v *master_pb.VolumeInformationMessage) bool { + matched, err := filepath.Match(collectionPattern, v.Collection) + if err != nil { + return false + } + return matched + } +} diff --git a/weed/shell/command_volume_copy.go b/weed/shell/command_volume_copy.go index 85b4bb095..0a4ac5063 100644 --- a/weed/shell/command_volume_copy.go +++ b/weed/shell/command_volume_copy.go @@ -3,6 +3,7 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -32,10 +33,6 @@ func (c *commandVolumeCopy) Help() string { func (c *commandVolumeCopy) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volCopyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volCopyCommand.Int("volumeId", 0, "the volume id") sourceNodeStr := volCopyCommand.String("source", "", "the source volume server :") @@ -44,7 +41,11 @@ func (c *commandVolumeCopy) Do(args []string, commandEnv *CommandEnv, writer io. return nil } - sourceVolumeServer, targetVolumeServer := *sourceNodeStr, *targetNodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer, targetVolumeServer := pb.ServerAddress(*sourceNodeStr), pb.ServerAddress(*targetNodeStr) volumeId := needle.VolumeId(*volumeIdInt) @@ -52,6 +53,6 @@ func (c *commandVolumeCopy) Do(args []string, commandEnv *CommandEnv, writer io. return fmt.Errorf("source and target volume servers are the same!") } - _, err = copyVolume(commandEnv.option.GrpcDialOption, volumeId, sourceVolumeServer, targetVolumeServer, "") + _, err = copyVolume(commandEnv.option.GrpcDialOption, writer, volumeId, sourceVolumeServer, targetVolumeServer, "") return } diff --git a/weed/shell/command_volume_delete.go b/weed/shell/command_volume_delete.go index 187caa1a4..30d71ba9f 100644 --- a/weed/shell/command_volume_delete.go +++ b/weed/shell/command_volume_delete.go @@ -2,6 +2,7 @@ package shell import ( "flag" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -30,10 +31,6 @@ func (c *commandVolumeDelete) Help() string { func (c *commandVolumeDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volDeleteCommand.Int("volumeId", 0, "the volume id") nodeStr := volDeleteCommand.String("node", "", "the volume server :") @@ -41,7 +38,11 @@ func (c *commandVolumeDelete) Do(args []string, commandEnv *CommandEnv, writer i return nil } - sourceVolumeServer := *nodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer := pb.ServerAddress(*nodeStr) volumeId := needle.VolumeId(*volumeIdInt) diff --git a/weed/shell/command_volume_delete_empty.go b/weed/shell/command_volume_delete_empty.go index 079915f66..79e61d38d 100644 --- a/weed/shell/command_volume_delete_empty.go +++ b/weed/shell/command_volume_delete_empty.go @@ -2,6 +2,7 @@ package shell import ( "flag" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "io" @@ -32,19 +33,20 @@ func (c *commandVolumeDeleteEmpty) Help() string { func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones") applyBalancing := volDeleteCommand.Bool("force", false, "apply to delete empty volumes") if err = volDeleteCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyBalancing, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -58,7 +60,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri if v.Size <= 8 && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds { if *applyBalancing { log.Printf("deleting empty volume %d from %s", v.Id, dn.Id) - if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil { + if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(dn)); deleteErr != nil { err = deleteErr } continue diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index efd5ae5de..4d986d252 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -4,11 +4,14 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "path/filepath" - "sort" + "strconv" + "time" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -51,58 +54,112 @@ func (c *commandVolumeFixReplication) Help() string { func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volFixReplicationCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes") retryCount := volFixReplicationCommand.Int("retry", 0, "how many times to retry") + volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle") + if err = volFixReplicationCommand.Parse(args); err != nil { return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + takeAction := !*skipChange - // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) - if err != nil { - return err - } + underReplicatedVolumeIdsCount := 1 + for underReplicatedVolumeIdsCount > 0 { + fixedVolumeReplicas := map[string]int{} - // find all volumes that needs replication - // collect all data nodes - volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo) + // collect topology information + topologyInfo, _, err := collectTopologyInfo(commandEnv, 15*time.Second) + if err != nil { + return err + } - if len(allLocations) == 0 { - return fmt.Errorf("no data nodes at all") - } + // find all volumes that needs replication + // collect all data nodes + volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo) - // find all under replicated volumes - var underReplicatedVolumeIds, overReplicatedVolumeIds []uint32 - for vid, replicas := range volumeReplicas { - replica := replicas[0] - replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement)) - if replicaPlacement.GetCopyCount() > len(replicas) { - underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid) - } else if replicaPlacement.GetCopyCount() < len(replicas) { - overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid) - fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas)) + if len(allLocations) == 0 { + return fmt.Errorf("no data nodes at all") } - } - if len(overReplicatedVolumeIds) > 0 { - return c.fixOverReplicatedVolumes(commandEnv, writer, takeAction, overReplicatedVolumeIds, volumeReplicas, allLocations) - } + // find all under replicated volumes + var underReplicatedVolumeIds, overReplicatedVolumeIds, misplacedVolumeIds []uint32 + for vid, replicas := range volumeReplicas { + replica := replicas[0] + replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement)) + if replicaPlacement.GetCopyCount() > len(replicas) { + underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid) + } else if replicaPlacement.GetCopyCount() < len(replicas) { + overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid) + fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas)) + } else if isMisplaced(replicas, replicaPlacement) { + misplacedVolumeIds = append(misplacedVolumeIds, vid) + fmt.Fprintf(writer, "volume %d replication %s is not well placed %+v\n", replica.info.Id, replicaPlacement, replicas) + } + } - if len(underReplicatedVolumeIds) == 0 { - return nil - } + if len(overReplicatedVolumeIds) > 0 { + if err := c.deleteOneVolume(commandEnv, writer, takeAction, overReplicatedVolumeIds, volumeReplicas, allLocations, pickOneReplicaToDelete); err != nil { + return err + } + } - // find the most under populated data nodes - return c.fixUnderReplicatedVolumes(commandEnv, writer, takeAction, underReplicatedVolumeIds, volumeReplicas, allLocations, *retryCount) + if len(misplacedVolumeIds) > 0 { + if err := c.deleteOneVolume(commandEnv, writer, takeAction, misplacedVolumeIds, volumeReplicas, allLocations, pickOneMisplacedVolume); err != nil { + return err + } + } + + underReplicatedVolumeIdsCount = len(underReplicatedVolumeIds) + if underReplicatedVolumeIdsCount > 0 { + // find the most under populated data nodes + fixedVolumeReplicas, err = c.fixUnderReplicatedVolumes(commandEnv, writer, takeAction, underReplicatedVolumeIds, volumeReplicas, allLocations, *retryCount, *volumesPerStep) + if err != nil { + return err + } + } + + if *skipChange { + break + } + // check that the topology has been updated + if len(fixedVolumeReplicas) > 0 { + fixedVolumes := make([]string, 0, len(fixedVolumeReplicas)) + for k, _ := range fixedVolumeReplicas { + fixedVolumes = append(fixedVolumes, k) + } + volumeIdLocations, err := lookupVolumeIds(commandEnv, fixedVolumes) + if err != nil { + return err + } + for _, volumeIdLocation := range volumeIdLocations { + volumeId := volumeIdLocation.VolumeOrFileId + volumeIdLocationCount := len(volumeIdLocation.Locations) + i := 0 + for fixedVolumeReplicas[volumeId] >= volumeIdLocationCount { + fmt.Fprintf(writer, "the number of locations for volume %s has not increased yet, let's wait\n", volumeId) + time.Sleep(time.Duration(i+1) * time.Second * 7) + volumeLocIds, err := lookupVolumeIds(commandEnv, []string{volumeId}) + if err != nil { + return err + } + volumeIdLocationCount = len(volumeLocIds[0].Locations) + if *retryCount <= i { + return fmt.Errorf("replicas volume %s mismatch in topology", volumeId) + } + i += 1 + } + } + } + } + return nil } func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo) (map[uint32][]*VolumeReplica, []location) { @@ -123,12 +180,14 @@ func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo) (map[ui return volumeReplicas, allLocations } -func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, overReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location) error { +type SelectOneVolumeFunc func(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica + +func (c *commandVolumeFixReplication) deleteOneVolume(commandEnv *CommandEnv, writer io.Writer, takeAction bool, overReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location, selectOneVolumeFn SelectOneVolumeFunc) error { for _, vid := range overReplicatedVolumeIds { replicas := volumeReplicas[vid] replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replicas[0].info.ReplicaPlacement)) - replica := pickOneReplicaToDelete(replicas, replicaPlacement) + replica := selectOneVolumeFn(replicas, replicaPlacement) // check collection name pattern if *c.collectionPattern != "" { @@ -141,13 +200,24 @@ func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *Comma } } + collectionIsMismatch := false + for _, volumeReplica := range replicas { + if volumeReplica.info.Collection != replica.info.Collection { + fmt.Fprintf(writer, "skip delete volume %d as collection %s is mismatch: %s\n", replica.info.Id, replica.info.Collection, volumeReplica.info.Collection) + collectionIsMismatch = true + } + } + if collectionIsMismatch { + continue + } + fmt.Fprintf(writer, "deleting volume %d from %s ...\n", replica.info.Id, replica.location.dataNode.Id) if !takeAction { break } - if err := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(replica.info.Id), replica.location.dataNode.Id); err != nil { + if err := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(replica.info.Id), pb.NewServerAddressFromDataNode(replica.location.dataNode)); err != nil { return fmt.Errorf("deleting volume %d from %s : %v", replica.info.Id, replica.location.dataNode.Id, err) } @@ -155,16 +225,22 @@ func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *Comma return nil } -func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, underReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location, retryCount int) (err error) { - +func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, underReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location, retryCount int, volumesPerStep int) (fixedVolumes map[string]int, err error) { + fixedVolumes = map[string]int{} + if len(underReplicatedVolumeIds) > volumesPerStep && volumesPerStep > 0 { + underReplicatedVolumeIds = underReplicatedVolumeIds[0:volumesPerStep] + } for _, vid := range underReplicatedVolumeIds { for i := 0; i < retryCount+1; i++ { if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil { + if takeAction { + fixedVolumes[strconv.FormatUint(uint64(vid), 10)] = len(volumeReplicas[vid]) + } break } } } - return + return fixedVolumes, nil } func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *CommandEnv, writer io.Writer, takeAction bool, volumeReplicas map[uint32][]*VolumeReplica, vid uint32, allLocations []location) error { @@ -200,14 +276,28 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co break } - err := operation.WithVolumeServerClient(dst.dataNode.Id, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{ + err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + stream, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{ VolumeId: replica.info.Id, - SourceDataNode: replica.location.dataNode.Id, + SourceDataNode: string(pb.NewServerAddressFromDataNode(replica.location.dataNode)), }) if replicateErr != nil { return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr) } + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + return recvErr + } + } + if resp.ProcessedBytes > 0 { + fmt.Fprintf(writer, "volume %d processed %d bytes\n", replica.info.Id, resp.ProcessedBytes) + } + } + return nil }) @@ -229,8 +319,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co func keepDataNodesSorted(dataNodes []location, diskType types.DiskType) { fn := capacityByFreeVolumeCount(diskType) - sort.Slice(dataNodes, func(i, j int) bool { - return fn(dataNodes[i].dataNode) > fn(dataNodes[j].dataNode) + slices.SortFunc(dataNodes, func(a, b location) bool { + return fn(a.dataNode) > fn(b.dataNode) }) } @@ -409,9 +499,7 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st } func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica { - - sort.Slice(replicas, func(i, j int) bool { - a, b := replicas[i], replicas[j] + slices.SortFunc(replicas, func(a, b *VolumeReplica) bool { if a.info.Size != b.info.Size { return a.info.Size < b.info.Size } @@ -427,3 +515,44 @@ func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_b return replicas[0] } + +// check and fix misplaced volumes + +func isMisplaced(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) bool { + + for i := 0; i < len(replicas); i++ { + others := otherThan(replicas, i) + if satisfyReplicaPlacement(replicaPlacement, others, *replicas[i].location) { + return false + } + } + + return true + +} + +func otherThan(replicas []*VolumeReplica, index int) (others []*VolumeReplica) { + for i := 0; i < len(replicas); i++ { + if index != i { + others = append(others, replicas[i]) + } + } + return +} + +func pickOneMisplacedVolume(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) (toDelete *VolumeReplica) { + + var deletionCandidates []*VolumeReplica + for i := 0; i < len(replicas); i++ { + others := otherThan(replicas, i) + if !isMisplaced(others, replicaPlacement) { + deletionCandidates = append(deletionCandidates, replicas[i]) + } + } + if len(deletionCandidates) > 0 { + return pickOneReplicaToDelete(deletionCandidates, replicaPlacement) + } + + return pickOneReplicaToDelete(replicas, replicaPlacement) + +} diff --git a/weed/shell/command_volume_fix_replication_test.go b/weed/shell/command_volume_fix_replication_test.go index bb61be1ef..5212fd2ed 100644 --- a/weed/shell/command_volume_fix_replication_test.go +++ b/weed/shell/command_volume_fix_replication_test.go @@ -261,6 +261,29 @@ func TestSatisfyReplicaPlacement00x(t *testing.T) { } +func TestSatisfyReplicaPlacement100(t *testing.T) { + + var tests = []testcase{ + { + name: "test 100", + replication: "100", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + possibleLocation: location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dn3"}}, + expected: true, + }, + } + + runTests(tests, t) + +} + func runTests(tests []testcase, t *testing.T) { for _, tt := range tests { replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication) @@ -271,3 +294,141 @@ func runTests(tests []testcase, t *testing.T) { } } } + +func TestMisplacedChecking(t *testing.T) { + + var tests = []testcase{ + { + name: "test 001", + replication: "001", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + expected: true, + }, + { + name: "test 010", + replication: "010", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + expected: false, + }, + { + name: "test 011", + replication: "011", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + expected: false, + }, + { + name: "test 110", + replication: "110", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + expected: false, + }, + { + name: "test 100", + replication: "100", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication) + println("replication:", tt.replication, "expected", tt.expected, "name:", tt.name) + if isMisplaced(tt.replicas, replicaPlacement) != tt.expected { + t.Errorf("%s: expect %v %v %+v", + tt.name, tt.expected, tt.replication, tt.replicas) + } + } + +} + +func TestPickingMisplacedVolumeToDelete(t *testing.T) { + + var tests = []testcase{ + { + name: "test 001", + replication: "001", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + info: &master_pb.VolumeInformationMessage{ + Size: 100, + }, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + info: &master_pb.VolumeInformationMessage{ + Size: 99, + }, + }, + }, + possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + { + name: "test 100", + replication: "100", + replicas: []*VolumeReplica{ + { + location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}, + info: &master_pb.VolumeInformationMessage{ + Size: 100, + }, + }, + { + location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + info: &master_pb.VolumeInformationMessage{ + Size: 99, + }, + }, + }, + possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}, + }, + } + + for _, tt := range tests { + replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication) + println("replication:", tt.replication, "name:", tt.name) + if x := pickOneMisplacedVolume(tt.replicas, replicaPlacement); x.location.dataNode.Id != tt.possibleLocation.dataNode.Id { + t.Errorf("%s: picked %+v for replication %v", + tt.name, x.location.dataNode.Id, tt.replication) + } else { + t.Logf("%s: picked %+v %v", + tt.name, x.location.dataNode.Id, tt.replication) + } + } + +} diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 9e778b918..2b1daf97c 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -5,22 +5,27 @@ import ( "context" "flag" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/needle" - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sync" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" + "io" + "io/ioutil" + "math" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" ) func init() { @@ -28,7 +33,8 @@ func init() { } type commandVolumeFsck struct { - env *CommandEnv + env *CommandEnv + forcePurging *bool } func (c *commandVolumeFsck) Name() string { @@ -57,23 +63,28 @@ func (c *commandVolumeFsck) Help() string { func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) verbose := fsckCommand.Bool("v", false, "verbose mode") findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") - applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " delete data not referenced by the filer") + findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") + applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer. Currently this only works with default filerGroup.") + c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging") + purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") + tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") + if err = fsckCommand.Parse(args); err != nil { return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + c.env = commandEnv // create a temp folder - tempFolder, err := ioutil.TempDir("", "sw_fsck") + tempFolder, err := os.MkdirTemp(*tempPath, "sw_fsck") if err != nil { return fmt.Errorf("failed to create temp folder: %v", err) } @@ -83,35 +94,63 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. defer os.RemoveAll(tempFolder) // collect all volume id locations - volumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer) + dataNodeVolumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer) if err != nil { return fmt.Errorf("failed to collect all volume locations: %v", err) } - // collect each volume file ids - for volumeId, vinfo := range volumeIdToVInfo { - err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose, writer) + isBucketsPath := false + var fillerBucketsPath string + if *findMissingChunksInFiler && *findMissingChunksInFilerPath != "/" { + fillerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { - return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err) + return fmt.Errorf("read filer buckets path: %v", err) + } + if strings.HasPrefix(*findMissingChunksInFilerPath, fillerBucketsPath) { + isBucketsPath = true + } + } + if err != nil { + return fmt.Errorf("read filer buckets path: %v", err) + } + + collectMtime := time.Now().Unix() + // collect each volume file ids + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + for volumeId, vinfo := range volumeIdToVInfo { + if *findMissingChunksInVolumeId > 0 && uint32(*findMissingChunksInVolumeId) != volumeId { + delete(volumeIdToVInfo, volumeId) + continue + } + if isBucketsPath && !strings.HasPrefix(*findMissingChunksInFilerPath, fillerBucketsPath+"/"+vinfo.collection) { + delete(volumeIdToVInfo, volumeId) + continue + } + err = c.collectOneVolumeFileIds(tempFolder, dataNodeId, volumeId, vinfo, *verbose, writer) + if err != nil { + return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err) + } } } if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, applyPurging); err != nil { + if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent, collectMtime); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } - // for each volume, check filer file ids - if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { - return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + // for each volume, check filer file ids + if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, dataNodeId, writer, *verbose, *applyPurging); err != nil { + return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) + } } } else { // collect all filer file ids - if err = c.collectFilerFileIds(tempFolder, volumeIdToVInfo, *verbose, writer); err != nil { + if err = c.collectFilerFileIds(dataNodeVolumeIdToVInfo, tempFolder, writer, *verbose); err != nil { return fmt.Errorf("failed to collect file ids from filer: %v", err) } - // volume file ids substract filer file ids - if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { + // volume file ids subtract filer file ids + if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) } } @@ -119,19 +158,24 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging *bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool, collectMtime int64) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") } files := make(map[uint32]*os.File) - for vid := range volumeIdToServer { - dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if openErr != nil { - return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + for _, volumeIdToServer := range dataNodeVolumeIdToVInfo { + for vid := range volumeIdToServer { + if _, ok := files[vid]; ok { + continue + } + dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if openErr != nil { + return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + } + files[vid] = dst } - files[vid] = dst } defer func() { for _, f := range files { @@ -145,31 +189,19 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint cookie uint32 path util.FullPath } - return doTraverseBfsAndSaving(c.env, nil, filerPath, false, func(outputChan chan interface{}) { - buffer := make([]byte, 16) - for item := range outputChan { - i := item.(*Item) - if f, ok := files[i.vid]; ok { - util.Uint64toBytes(buffer, i.fileKey) - util.Uint32toBytes(buffer[8:], i.cookie) - util.Uint32toBytes(buffer[12:], uint32(len(i.path))) - f.Write(buffer) - f.Write([]byte(i.path)) - // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path) - } else { - fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) - } - } - }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { + return doTraverseBfsAndSaving(c.env, nil, filerPath, false, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if verbose && entry.Entry.IsDirectory { fmt.Fprintf(writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name)) } - dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) if resolveErr != nil { return nil } - dChunks = append(dChunks, mChunks...) - for _, chunk := range dChunks { + dataChunks = append(dataChunks, manifestChunks...) + for _, chunk := range dataChunks { + if chunk.Mtime > collectMtime { + continue + } outputChan <- &Item{ vid: chunk.Fid.VolumeId, fileKey: chunk.Fid.FileKey, @@ -178,15 +210,33 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint } } return nil + }, func(outputChan chan interface{}) { + buffer := make([]byte, 16) + for item := range outputChan { + i := item.(*Item) + if f, ok := files[i.vid]; ok { + util.Uint64toBytes(buffer, i.fileKey) + util.Uint32toBytes(buffer[8:], i.cookie) + util.Uint32toBytes(buffer[12:], uint32(len(i.path))) + f.Write(buffer) + f.Write([]byte(i.path)) + // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path) + } else { + fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) + if purgeAbsent { + fmt.Printf("deleting path %s after volume not found", i.path) + c.httpDelete(i.path, verbose) + } + } + } }) - return nil } -func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { +func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, dataNodeId string, writer io.Writer, verbose bool, applyPurging bool) error { for volumeId, vinfo := range volumeIdToVInfo { - checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose) + checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, dataNodeId, volumeId, writer, verbose, applyPurging) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -194,61 +244,120 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf return nil } -func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { - var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 - for volumeId, vinfo := range volumeIdToVInfo { - inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose) - if checkErr != nil { - return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) - } - totalInUseCount += inUseCount - totalOrphanChunkCount += uint64(len(orphanFileIds)) - totalOrphanDataSize += orphanDataSize +func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { - if verbose { + var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 + volumeIdOrphanFileIds := make(map[uint32]map[string]bool) + isSeveralReplicas := make(map[uint32]bool) + isEcVolumeReplicas := make(map[uint32]bool) + isReadOnlyReplicas := make(map[uint32]bool) + serverReplicas := make(map[uint32][]pb.ServerAddress) + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + for volumeId, vinfo := range volumeIdToVInfo { + inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, dataNodeId, volumeId, writer, verbose) + if checkErr != nil { + return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) + } + isSeveralReplicas[volumeId] = false + if _, found := volumeIdOrphanFileIds[volumeId]; !found { + volumeIdOrphanFileIds[volumeId] = make(map[string]bool) + } else { + isSeveralReplicas[volumeId] = true + } for _, fid := range orphanFileIds { - fmt.Fprintf(writer, "%s\n", fid) + if isSeveralReplicas[volumeId] { + if _, found := volumeIdOrphanFileIds[volumeId][fid]; !found { + continue + } + } + volumeIdOrphanFileIds[volumeId][fid] = isSeveralReplicas[volumeId] + } + + totalInUseCount += inUseCount + totalOrphanChunkCount += uint64(len(orphanFileIds)) + totalOrphanDataSize += orphanDataSize + + if verbose { + for _, fid := range orphanFileIds { + fmt.Fprintf(writer, "%s\n", fid) + } } + isEcVolumeReplicas[volumeId] = vinfo.isEcVolume + if isReadOnly, found := isReadOnlyReplicas[volumeId]; !(found && isReadOnly) { + isReadOnlyReplicas[volumeId] = vinfo.isReadOnly + } + serverReplicas[volumeId] = append(serverReplicas[volumeId], vinfo.server) } - if *applyPurging && len(orphanFileIds) > 0 { - if vinfo.isEcVolume { - fmt.Fprintf(writer, "Skip purging for Erasure Coded volumes.\n") + for volumeId, orphanReplicaFileIds := range volumeIdOrphanFileIds { + if !(applyPurging && len(orphanReplicaFileIds) > 0) { + continue } - if inUseCount == 0 { - if err := deleteVolume(c.env.option.GrpcDialOption, needle.VolumeId(volumeId), vinfo.server); err != nil { - return fmt.Errorf("delete volume %d: %v\n", volumeId, err) + orphanFileIds := []string{} + for fid, foundInAllReplicas := range orphanReplicaFileIds { + if !isSeveralReplicas[volumeId] || *c.forcePurging || (isSeveralReplicas[volumeId] && foundInAllReplicas) { + orphanFileIds = append(orphanFileIds, fid) } - } else { + } + if !(len(orphanFileIds) > 0) { + continue + } + if verbose { + fmt.Fprintf(writer, "purging process for volume %d.\n", volumeId) + } + + if isEcVolumeReplicas[volumeId] { + fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) + continue + } + for _, server := range serverReplicas[volumeId] { + needleVID := needle.VolumeId(volumeId) + + if isReadOnlyReplicas[volumeId] { + err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, true) + if err != nil { + return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) + } + + fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, server) + defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, false) + + fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, server) + } + if verbose { + fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) + } + if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { - return fmt.Errorf("purge for volume %d: %v\n", volumeId, err) + return fmt.Errorf("purging volume %d: %v", volumeId, err) } } } } - if totalOrphanChunkCount == 0 { - fmt.Fprintf(writer, "no orphan data\n") - return nil - } - - if !*applyPurging { + if !applyPurging { pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount)) fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize) fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n") } + + if totalOrphanChunkCount == 0 { + fmt.Fprintf(writer, "no orphan data\n") + //return nil + } + return nil } -func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error { +func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeId string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error { if verbose { fmt.Fprintf(writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server) } - return operation.WithVolumeServerClient(vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { ext := ".idx" if vinfo.isEcVolume { @@ -268,7 +377,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err) } - err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId)) + err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)) if err != nil { return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) } @@ -279,19 +388,21 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId } -func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo, verbose bool, writer io.Writer) error { +func (c *commandVolumeFsck) collectFilerFileIds(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool) error { if verbose { fmt.Fprintf(writer, "collecting file ids from filer ...\n") } files := make(map[uint32]*os.File) - for vid := range volumeIdToServer { - dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if openErr != nil { - return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + for _, volumeIdToServer := range dataNodeVolumeIdToVInfo { + for vid := range volumeIdToServer { + dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if openErr != nil { + return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + } + files[vid] = dst } - files[vid] = dst } defer func() { for _, f := range files { @@ -303,42 +414,42 @@ func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToSer vid uint32 fileKey uint64 } - return doTraverseBfsAndSaving(c.env, nil, "/", false, func(outputChan chan interface{}) { - buffer := make([]byte, 8) - for item := range outputChan { - i := item.(*Item) - util.Uint64toBytes(buffer, i.fileKey) - files[i.vid].Write(buffer) - } - }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { - dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) + return doTraverseBfsAndSaving(c.env, nil, "/", false, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) if resolveErr != nil { if verbose { fmt.Fprintf(writer, "resolving manifest chunks in %s: %v\n", util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr) } return nil } - dChunks = append(dChunks, mChunks...) - for _, chunk := range dChunks { + dataChunks = append(dataChunks, manifestChunks...) + for _, chunk := range dataChunks { outputChan <- &Item{ vid: chunk.Fid.VolumeId, fileKey: chunk.Fid.FileKey, } } return nil + }, func(outputChan chan interface{}) { + buffer := make([]byte, 8) + for item := range outputChan { + i := item.(*Item) + util.Uint64toBytes(buffer, i.fileKey) + files[i.vid].Write(buffer) + } }) } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, dataNodeId string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { if verbose { - fmt.Fprintf(writer, "find missing file chuns in volume %d ...\n", volumeId) + fmt.Fprintf(writer, "find missing file chunks in dataNodeId %s volume %d ...\n", dataNodeId, volumeId) } db := needle_map.NewMemDb() defer db.Close() - if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil { + if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)); err != nil { return } @@ -362,11 +473,7 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo for { readSize, err = io.ReadFull(br, buffer) if err != nil || readSize != 16 { - if err == io.EOF { - return nil - } else { - break - } + break } item.fileKey = util.BytesToUint64(buffer[:8]) @@ -382,26 +489,63 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo } item.path = util.FullPath(string(pathBytes)) - if _, found := db.Get(types.NeedleId(item.fileKey)); !found { - fmt.Fprintf(writer, "%d,%x%08x in %s %d not found\n", volumeId, item.fileKey, item.cookie, item.path, pathSize) + needleId := types.NeedleId(item.fileKey) + if _, found := db.Get(needleId); !found { + fmt.Fprintf(writer, "%s\n", item.path) + + if applyPurging { + // defining the URL this way automatically escapes complex path names + c.httpDelete(item.path, verbose) + } } + } + return nil +} +func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) { + req, err := http.NewRequest(http.MethodDelete, "", nil) + + req.URL = &url.URL{ + Scheme: "http", + Host: c.env.option.FilerAddress.ToHttpAddress(), + Path: string(path), + } + if verbose { + fmt.Printf("full HTTP delete request to be sent: %v\n", req) + } + if err != nil { + fmt.Errorf("HTTP delete request error: %v\n", err) } - return + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + fmt.Errorf("DELETE fetch error: %v\n", err) + } + defer resp.Body.Close() + + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Errorf("DELETE response error: %v\n", err) + } + + if verbose { + fmt.Println("delete response Status : ", resp.Status) + fmt.Println("delete response Headers : ", resp.Header) + } } -func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, dataNodeId string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { db := needle_map.NewMemDb() defer db.Close() - if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil { + if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)); err != nil { return } - filerFileIdsData, err := ioutil.ReadFile(getFilerFileIdFile(tempFolder, volumeId)) + filerFileIdsData, err := os.ReadFile(getFilerFileIdFile(tempFolder, volumeId)) if err != nil { return } @@ -428,8 +572,8 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder stri if orphanFileCount > 0 { pct := float64(orphanFileCount*100) / (float64(orphanFileCount + inUseCount)) - fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", - volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize) + fmt.Fprintf(writer, "dataNode:%s\tvolume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", + dataNodeId, volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize) } return @@ -437,38 +581,43 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder stri } type VInfo struct { - server string + server pb.ServerAddress collection string isEcVolume bool + isReadOnly bool } -func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[uint32]VInfo, err error) { +func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[string]map[uint32]VInfo, err error) { if verbose { fmt.Fprintf(writer, "collecting volume id and locations from master ...\n") } - volumeIdToServer = make(map[uint32]VInfo) + volumeIdToServer = make(map[string]map[uint32]VInfo) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } eachDataNode(topologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) { for _, diskInfo := range t.DiskInfos { + dataNodeId := t.GetId() + volumeIdToServer[dataNodeId] = make(map[uint32]VInfo) for _, vi := range diskInfo.VolumeInfos { - volumeIdToServer[vi.Id] = VInfo{ - server: t.Id, + volumeIdToServer[dataNodeId][vi.Id] = VInfo{ + server: pb.NewServerAddressFromDataNode(t), collection: vi.Collection, isEcVolume: false, + isReadOnly: vi.ReadOnly, } } for _, ecShardInfo := range diskInfo.EcShardInfos { - volumeIdToServer[ecShardInfo.Id] = VInfo{ - server: t.Id, + volumeIdToServer[dataNodeId][ecShardInfo.Id] = VInfo{ + server: pb.NewServerAddressFromDataNode(t), collection: ecShardInfo.Collection, isEcVolume: true, + isReadOnly: true, } } } @@ -491,7 +640,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] var wg sync.WaitGroup for _, location := range locations { wg.Add(1) - go func(server string, fidList []string) { + go func(server pb.ServerAddress, fidList []string) { defer wg.Done() if deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil { @@ -500,7 +649,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] resultChan <- deleteResults } - }(location.Url, fileIds) + }(location.ServerAddress(), fileIds) } wg.Wait() close(resultChan) @@ -516,8 +665,8 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] return } -func getVolumeFileIdFile(tempFolder string, vid uint32) string { - return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid)) +func getVolumeFileIdFile(tempFolder string, dataNodeid string, vid uint32) string { + return filepath.Join(tempFolder, fmt.Sprintf("%s_%d.idx", dataNodeid, vid)) } func getFilerFileIdFile(tempFolder string, vid uint32) string { diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go index 9856de10b..3a5633168 100644 --- a/weed/shell/command_volume_list.go +++ b/weed/shell/command_volume_list.go @@ -2,12 +2,14 @@ package shell import ( "bytes" + "flag" "fmt" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "golang.org/x/exp/slices" + "path/filepath" "io" - "sort" ) func init() { @@ -15,6 +17,9 @@ func init() { } type commandVolumeList struct { + collectionPattern *string + readonly *bool + volumeId *uint64 } func (c *commandVolumeList) Name() string { @@ -31,13 +36,23 @@ func (c *commandVolumeList) Help() string { func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + volumeListCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + verbosityLevel := volumeListCommand.Int("v", 5, "verbose mode: 0, 1, 2, 3, 4, 5") + c.collectionPattern = volumeListCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") + c.readonly = volumeListCommand.Bool("readonly", false, "show only readonly") + c.volumeId = volumeListCommand.Uint64("volumeId", 0, "show only volume id") + + if err = volumeListCommand.Parse(args); err != nil { + return nil + } + // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } - writeTopologyInfo(writer, topologyInfo, volumeSizeLimitMb) + c.writeTopologyInfo(writer, topologyInfo, volumeSizeLimitMb, *verbosityLevel) return nil } @@ -58,77 +73,107 @@ func diskInfoToString(diskInfo *master_pb.DiskInfo) string { return buf.String() } -func writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLimitMb uint64) statistics { - fmt.Fprintf(writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(t.DiskInfos)) - sort.Slice(t.DataCenterInfos, func(i, j int) bool { - return t.DataCenterInfos[i].Id < t.DataCenterInfos[j].Id +func (c *commandVolumeList) writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLimitMb uint64, verbosityLevel int) statistics { + output(verbosityLevel >= 0, writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(t.DiskInfos)) + slices.SortFunc(t.DataCenterInfos, func(a, b *master_pb.DataCenterInfo) bool { + return a.Id < b.Id }) var s statistics for _, dc := range t.DataCenterInfos { - s = s.plus(writeDataCenterInfo(writer, dc)) + s = s.plus(c.writeDataCenterInfo(writer, dc, verbosityLevel)) } - fmt.Fprintf(writer, "%+v \n", s) + output(verbosityLevel >= 0, writer, "%+v \n", s) return s } -func writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo) statistics { - fmt.Fprintf(writer, " DataCenter %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) + +func (c *commandVolumeList) writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo, verbosityLevel int) statistics { + output(verbosityLevel >= 1, writer, " DataCenter %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics - sort.Slice(t.RackInfos, func(i, j int) bool { - return t.RackInfos[i].Id < t.RackInfos[j].Id + slices.SortFunc(t.RackInfos, func(a, b *master_pb.RackInfo) bool { + return a.Id < b.Id }) for _, r := range t.RackInfos { - s = s.plus(writeRackInfo(writer, r)) + s = s.plus(c.writeRackInfo(writer, r, verbosityLevel)) } - fmt.Fprintf(writer, " DataCenter %s %+v \n", t.Id, s) + output(verbosityLevel >= 1, writer, " DataCenter %s %+v \n", t.Id, s) return s } -func writeRackInfo(writer io.Writer, t *master_pb.RackInfo) statistics { - fmt.Fprintf(writer, " Rack %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) + +func (c *commandVolumeList) writeRackInfo(writer io.Writer, t *master_pb.RackInfo, verbosityLevel int) statistics { + output(verbosityLevel >= 2, writer, " Rack %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics - sort.Slice(t.DataNodeInfos, func(i, j int) bool { - return t.DataNodeInfos[i].Id < t.DataNodeInfos[j].Id + slices.SortFunc(t.DataNodeInfos, func(a, b *master_pb.DataNodeInfo) bool { + return a.Id < b.Id }) for _, dn := range t.DataNodeInfos { - s = s.plus(writeDataNodeInfo(writer, dn)) + s = s.plus(c.writeDataNodeInfo(writer, dn, verbosityLevel)) } - fmt.Fprintf(writer, " Rack %s %+v \n", t.Id, s) + output(verbosityLevel >= 2, writer, " Rack %s %+v \n", t.Id, s) return s } -func writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo) statistics { - fmt.Fprintf(writer, " DataNode %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) + +func (c *commandVolumeList) writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo, verbosityLevel int) statistics { + output(verbosityLevel >= 3, writer, " DataNode %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics for _, diskInfo := range t.DiskInfos { - s = s.plus(writeDiskInfo(writer, diskInfo)) + s = s.plus(c.writeDiskInfo(writer, diskInfo, verbosityLevel)) } - fmt.Fprintf(writer, " DataNode %s %+v \n", t.Id, s) + output(verbosityLevel >= 3, writer, " DataNode %s %+v \n", t.Id, s) return s } -func writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo) statistics { +func (c *commandVolumeList) isNotMatchDiskInfo(readOnly bool, collection string, volumeId uint32) bool { + if *c.readonly && !readOnly { + return true + } + if *c.collectionPattern != "" { + if matched, _ := filepath.Match(*c.collectionPattern, collection); !matched { + return true + } + } + if *c.volumeId > 0 && *c.volumeId != uint64(volumeId) { + return true + } + return false +} + +func (c *commandVolumeList) writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo, verbosityLevel int) statistics { var s statistics diskType := t.Type if diskType == "" { diskType = "hdd" } - fmt.Fprintf(writer, " Disk %s(%s)\n", diskType, diskInfoToString(t)) - sort.Slice(t.VolumeInfos, func(i, j int) bool { - return t.VolumeInfos[i].Id < t.VolumeInfos[j].Id + output(verbosityLevel >= 4, writer, " Disk %s(%s)\n", diskType, diskInfoToString(t)) + slices.SortFunc(t.VolumeInfos, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Id < b.Id }) for _, vi := range t.VolumeInfos { - s = s.plus(writeVolumeInformationMessage(writer, vi)) + if c.isNotMatchDiskInfo(vi.ReadOnly, vi.Collection, vi.Id) { + continue + } + s = s.plus(writeVolumeInformationMessage(writer, vi, verbosityLevel)) } for _, ecShardInfo := range t.EcShardInfos { - fmt.Fprintf(writer, " ec volume id:%v collection:%v shards:%v\n", ecShardInfo.Id, ecShardInfo.Collection, erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds()) + if c.isNotMatchDiskInfo(false, ecShardInfo.Collection, ecShardInfo.Id) { + continue + } + output(verbosityLevel >= 5, writer, " ec volume id:%v collection:%v shards:%v\n", ecShardInfo.Id, ecShardInfo.Collection, erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds()) } - fmt.Fprintf(writer, " Disk %s %+v \n", diskType, s) + output(verbosityLevel >= 4, writer, " Disk %s %+v \n", diskType, s) return s } -func writeVolumeInformationMessage(writer io.Writer, t *master_pb.VolumeInformationMessage) statistics { - fmt.Fprintf(writer, " volume %+v \n", t) +func writeVolumeInformationMessage(writer io.Writer, t *master_pb.VolumeInformationMessage, verbosityLevel int) statistics { + output(verbosityLevel >= 5, writer, " volume %+v \n", t) return newStatistics(t) } +func output(condition bool, w io.Writer, format string, a ...interface{}) { + if condition { + fmt.Fprintf(w, format, a...) + } +} + type statistics struct { Size uint64 FileCount uint64 diff --git a/weed/shell/command_volume_list_test.go b/weed/shell/command_volume_list_test.go index 379cf4943..0bcbb1713 100644 --- a/weed/shell/command_volume_list_test.go +++ b/weed/shell/command_volume_list_test.go @@ -1,6 +1,7 @@ package shell import ( + _ "embed" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" @@ -64,11 +65,14 @@ func parseOutput(output string) *master_pb.TopologyInfo { case "Disk": if disk == nil { diskType := parts[1][:strings.Index(parts[1], "(")] + volumeCountStr := parts[1][strings.Index(parts[1], ":")+1 : strings.Index(parts[1], "/")] maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:] maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr) + volumeCount, _ := strconv.Atoi(volumeCountStr) disk = &master_pb.DiskInfo{ Type: diskType, MaxVolumeCount: int64(maxVolumeCount), + VolumeCount: int64(volumeCount), } dn.DiskInfos[types.ToDiskType(diskType).String()] = disk } else { @@ -85,809 +89,5 @@ func parseOutput(output string) *master_pb.TopologyInfo { return topo } -const topoData = ` -Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) - DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0) - Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0) - Rack DefaultRack total size:0 file_count:0 - DataCenter dc1 total size:0 file_count:0 - DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0) - Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0) - DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0) - Disk hdd(volume:50/240 active:50 free:190 remote:0) - volume id:15 size:1115965064 collection:"collection0" file_count:83 replica_placement:100 version:3 modified_at_second:1609923671 - volume id:21 size:1097631536 collection:"collection0" file_count:82 delete_count:7 deleted_byte_count:68975485 replica_placement:100 version:3 modified_at_second:1609929578 - volume id:22 size:1086828272 collection:"collection0" file_count:75 replica_placement:100 version:3 modified_at_second:1609930001 - volume id:23 size:1076380216 collection:"collection0" file_count:68 replica_placement:100 version:3 modified_at_second:1609930434 - volume id:24 size:1074139776 collection:"collection0" file_count:90 replica_placement:100 version:3 modified_at_second:1609930909 - volume id:25 size:690757512 collection:"collection0" file_count:38 replica_placement:100 version:3 modified_at_second:1611144216 - volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 - volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 - volume id:29 size:281582680 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 - volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 - volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 - volume id:33 size:1130226400 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 - volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 - volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 - volume id:52 size:1083529728 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 - volume id:54 size:1045022344 collection:"collection1" file_count:9408 delete_count:30 deleted_byte_count:15132106 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:63 size:956941112 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 - volume id:69 size:869213648 collection:"collection1" file_count:7293 delete_count:102 deleted_byte_count:30643207 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 - volume id:74 size:957046128 collection:"collection1" file_count:6982 delete_count:258 deleted_byte_count:73054259 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 - volume id:80 size:827912928 collection:"collection1" file_count:6914 delete_count:17 deleted_byte_count:5689635 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631157 - volume id:84 size:873121856 collection:"collection1" file_count:8200 delete_count:13 deleted_byte_count:3131676 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 - volume id:85 size:1023869320 collection:"collection1" file_count:7788 delete_count:234 deleted_byte_count:78037967 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 - volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631193 - volume id:98 size:1077836440 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 - volume id:105 size:1073996824 collection:"collection1" file_count:6872 delete_count:20 deleted_byte_count:14482293 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499757 - volume id:106 size:1075458664 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 - volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 - volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611565 - volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573878 - volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 - volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584779 - volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 - volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 - volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 - volume id:161 size:1077397192 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 - volume id:173 size:1074154704 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2578509 replica_placement:100 version:3 modified_at_second:1615591904 - volume id:174 size:1073824232 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:197 size:1075423240 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485254 - volume id:219 size:1092298904 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 - volume id:263 size:1077167352 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 - volume id:272 size:1076146040 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 - volume id:291 size:1076256760 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:299 size:1075147824 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918454 - volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 - volume id:302 size:1077559792 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 - volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969996 - volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977458 - volume id:355 size:1075239792 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 - volume id:373 size:1080928000 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016877 - Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0) - DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0) - Disk hdd(volume:36/190 active:36 free:154 remote:0) - volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:480564 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 - volume id:3 size:308743136 file_count:1638 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 - volume id:4 size:285986968 file_count:1641 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 - volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 - volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 - volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 - volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814550 - volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819732 - volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827837 - volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956980 - volume id:16 size:1113666104 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:21 size:1097631664 collection:"collection0" file_count:82 delete_count:11 deleted_byte_count:68985100 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:56 size:1001897616 collection:"collection1" file_count:8762 delete_count:37 deleted_byte_count:65375405 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 - volume id:81 size:880693104 collection:"collection1" file_count:7481 delete_count:236 deleted_byte_count:80386421 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631396 - volume id:104 size:1076383624 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100728071 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602658 - volume id:107 size:1073811840 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747484 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 - volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 - volume id:139 size:1074163936 collection:"collection1" file_count:9315 delete_count:42 deleted_byte_count:10630966 replica_placement:100 version:3 modified_at_second:1615571946 - volume id:151 size:1098659752 collection:"collection1" file_count:10808 delete_count:24 deleted_byte_count:7088102 replica_placement:100 version:3 modified_at_second:1615586389 - volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:9076141 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:167 size:1073958176 collection:"collection1" file_count:25229 delete_count:48 deleted_byte_count:25871565 replica_placement:100 version:3 modified_at_second:1615602669 - volume id:177 size:1074120216 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3803952 replica_placement:100 version:3 modified_at_second:1615516892 - volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45552859 replica_placement:100 version:3 modified_at_second:1615580308 - volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1452346 replica_placement:100 version:3 modified_at_second:1615568922 - volume id:215 size:1068268216 collection:"collection1" file_count:2813 delete_count:10 deleted_byte_count:5676795 replica_placement:100 version:3 modified_at_second:1615586386 - volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 - volume id:283 size:1080178944 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896626 - volume id:303 size:1075944504 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 - volume id:309 size:1075178624 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 - volume id:323 size:1074608200 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 - volume id:344 size:1075035448 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 - volume id:347 size:1075145496 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 - volume id:357 size:1074276208 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 - volume id:380 size:1010760456 collection:"collection2" file_count:14921 delete_count:6 deleted_byte_count:65678 replica_placement:100 version:3 modified_at_second:1615632322 - volume id:381 size:939292792 collection:"collection2" file_count:14619 delete_count:2 deleted_byte_count:5119 replica_placement:100 version:3 modified_at_second:1615632324 - Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 - DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0) - Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0) - DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0) - Disk hdd(volume:108/1850 active:108 free:1742 remote:0) - volume id:1 size:284685936 file_count:1557 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 - volume id:32 size:281390512 file_count:1496 delete_count:6 deleted_byte_count:546403 replica_placement:100 version:3 modified_at_second:1615632362 - volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 - volume id:49 size:1078775288 collection:"collection1" file_count:9636 delete_count:22 deleted_byte_count:5625976 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630446 - volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:88 size:1073767976 collection:"collection1" file_count:14995 delete_count:206 deleted_byte_count:81222360 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629897 - volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 - volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 - volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 - volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615564274 - volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 - volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 - volume id:212 size:1078293360 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:213 size:1093587976 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 - volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 - volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 - volume id:227 size:1106699896 collection:"collection1" file_count:2827 delete_count:20 deleted_byte_count:5496790 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:229 size:1109855312 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609988 - volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 - volume id:231 size:1112917696 collection:"collection1" file_count:3151 delete_count:20 deleted_byte_count:2989828 replica_placement:100 version:3 modified_at_second:1615611350 - volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611352 - volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 - volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630838 - volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 - volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1614852171 - volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 - volume id:246 size:1075998648 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852105 - volume id:248 size:1084301184 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 - volume id:249 size:1074819136 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856230 - volume id:251 size:1075684488 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 - volume id:252 size:1075065208 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861196 - volume id:253 size:1087328816 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861255 - volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 - volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 - volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:259 size:1075507776 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 - volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 - volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1615629130 - volume id:266 size:1075617464 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 - volume id:267 size:1075699544 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 - volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877434 - volume id:269 size:1077552872 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 - volume id:270 size:1076876568 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 - volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 - volume id:277 size:1074956288 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 - volume id:278 size:1078798640 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 - volume id:279 size:1077325040 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890229 - volume id:280 size:1077432216 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 - volume id:281 size:1077581096 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 - volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 - volume id:285 size:1082128688 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 - volume id:289 size:1075284256 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 - volume id:290 size:1074723792 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:291 size:1076256768 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:293 size:1075409792 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 - volume id:294 size:1075444048 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 - volume id:296 size:1077824032 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 - volume id:297 size:1080229136 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 - volume id:298 size:1075410136 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 - volume id:299 size:1075147936 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918455 - volume id:300 size:1076212392 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 - volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 - volume id:303 size:1075944480 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 - volume id:306 size:1074764016 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 - volume id:307 size:1076568000 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 - volume id:308 size:1074022392 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 - volume id:309 size:1075178664 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 - volume id:310 size:1074761528 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 - volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 - volume id:315 size:1084153544 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937885 - volume id:317 size:1076215096 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 - volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 - volume id:319 size:1073952880 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 - volume id:320 size:1082437792 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943599 - volume id:321 size:1081477904 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 - volume id:324 size:1075606680 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 - volume id:325 size:1080701144 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 - volume id:330 size:1080825832 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956477 - volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 - volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 - volume id:336 size:1087853056 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963005 - volume id:337 size:1075646784 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 - volume id:338 size:1076118304 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 - volume id:340 size:1079462184 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969978 - volume id:341 size:1074448400 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 - volume id:342 size:1080186424 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977521 - volume id:344 size:1075035416 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 - volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977457 - volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1615631322 - volume id:348 size:1080623640 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984606 - volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 - volume id:351 size:1078795112 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 - volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 - volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 - volume id:355 size:1075239728 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 - volume id:356 size:1083305048 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992028 - volume id:358 size:1085152368 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 - volume id:360 size:1075532456 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 - volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 - volume id:375 size:1076140568 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 - volume id:376 size:1074845944 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 - volume id:377 size:957284144 collection:"collection2" file_count:14923 delete_count:1 deleted_byte_count:1797 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:378 size:959273936 collection:"collection2" file_count:15027 delete_count:4 deleted_byte_count:231414 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:381 size:939261032 collection:"collection2" file_count:14615 delete_count:5 deleted_byte_count:1192272 replica_placement:100 version:3 modified_at_second:1615632324 - Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0) - Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0) - DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0) - Disk hdd(volume:267/2000 active:267 free:1733 remote:0) - volume id:1 size:284693256 file_count:1558 delete_count:2 deleted_byte_count:4818 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 - volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:464508 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 - volume id:3 size:308741952 file_count:1637 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 - volume id:4 size:285986968 file_count:1640 delete_count:1 deleted_byte_count:145095 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 - volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 - volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 - volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 - volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 - volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814543 - volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1609815123 - volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819726 - volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827832 - volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956983 - volume id:15 size:1115965160 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:4956 replica_placement:100 version:3 modified_at_second:1612957001 - volume id:16 size:1113666048 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1612957011 - volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1612957011 - volume id:22 size:1086828368 collection:"collection0" file_count:75 delete_count:3 deleted_byte_count:5551 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:23 size:1076380280 collection:"collection0" file_count:68 delete_count:2 deleted_byte_count:2910 replica_placement:100 version:3 modified_at_second:1612957011 - volume id:24 size:1074139808 collection:"collection0" file_count:90 delete_count:1 deleted_byte_count:1977 replica_placement:100 version:3 modified_at_second:1612957011 - volume id:25 size:690757544 collection:"collection0" file_count:38 delete_count:1 deleted_byte_count:1944 replica_placement:100 version:3 modified_at_second:1612956995 - volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170023 - volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 - volume id:35 size:1075241656 collection:"collection1" file_count:10523 delete_count:96 deleted_byte_count:46618989 replica_placement:100 version:3 modified_at_second:1615618790 - volume id:36 size:1075118360 collection:"collection1" file_count:10342 delete_count:116 deleted_byte_count:25493106 replica_placement:100 version:3 modified_at_second:1615606148 - volume id:37 size:1075895584 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594777 - volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 - volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 - volume id:41 size:1076283528 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19319268 replica_placement:100 version:3 modified_at_second:1615596736 - volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548908 - volume id:43 size:1116445864 collection:"collection1" file_count:7358 delete_count:54 deleted_byte_count:9534379 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573499 - volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569826 - volume id:48 size:1076778720 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 - volume id:50 size:1076688224 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610879 - volume id:52 size:1083529704 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 - volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631452 - volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631774 - volume id:58 size:908064200 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632342 - volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 - volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 - volume id:61 size:975604488 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 - volume id:62 size:873845936 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 - volume id:64 size:965638488 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631031 - volume id:65 size:823283552 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632306 - volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 - volume id:67 size:878713872 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 - volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630982 - volume id:73 size:929047664 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 - volume id:74 size:957046128 collection:"collection1" file_count:6981 delete_count:259 deleted_byte_count:73080838 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 - volume id:75 size:908044992 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 - volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632013 - volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 - volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 - volume id:82 size:1041661800 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 - volume id:83 size:936195856 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 - volume id:85 size:1023867520 collection:"collection1" file_count:7787 delete_count:240 deleted_byte_count:82091742 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 - volume id:86 size:1009437488 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:87 size:922276640 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 - volume id:89 size:1044401976 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 - volume id:90 size:891145784 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629390 - volume id:91 size:936572832 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 - volume id:92 size:992440712 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 - volume id:93 size:1079603768 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556015 - volume id:94 size:1030685824 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 - volume id:95 size:990879168 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631866 - volume id:96 size:989296136 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 - volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631194 - volume id:99 size:1071718504 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 - volume id:100 size:1083617440 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505917 - volume id:101 size:1077109520 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 - volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626683 - volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 - volume id:104 size:1076383768 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100578087 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602661 - volume id:105 size:1073996824 collection:"collection1" file_count:6873 delete_count:19 deleted_byte_count:14271533 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499756 - volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 - volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 - volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 - volume id:111 size:1073826232 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593233 - volume id:114 size:1074762784 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 - volume id:115 size:1076192240 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285295 - volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611567 - volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573714 - volume id:118 size:1074064400 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516265 - volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573880 - volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 - volume id:121 size:1078803248 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506629 - volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:126 size:1073867464 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585912 - volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612991 - volume id:129 size:1074704272 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 - volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582330 - volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 - volume id:132 size:1088539496 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619584 - volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584780 - volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555876 - volume id:135 size:1073906720 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 - volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:139 size:1074163936 collection:"collection1" file_count:9314 delete_count:43 deleted_byte_count:10631353 replica_placement:100 version:3 modified_at_second:1615571946 - volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 - volume id:142 size:1075732992 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488586 - volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 - volume id:146 size:1076337520 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 - volume id:147 size:1077130544 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604117 - volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586393 - volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494496 - volume id:150 size:1076290408 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611569 - volume id:151 size:1098659752 collection:"collection1" file_count:10805 delete_count:27 deleted_byte_count:7209106 replica_placement:100 version:3 modified_at_second:1615586390 - volume id:152 size:1075941376 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:154 size:1081244752 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478471 - volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628342 - volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 - volume id:159 size:1074132336 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 - volume id:160 size:1075680976 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:161 size:1077397136 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 - volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478127 - volume id:163 size:1074457224 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 - volume id:164 size:1074261256 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620085 - volume id:165 size:1073891080 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471910 - volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491021 - volume id:167 size:1073958280 collection:"collection1" file_count:25231 delete_count:48 deleted_byte_count:26022344 replica_placement:100 version:3 modified_at_second:1615620014 - volume id:168 size:1074718864 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 - volume id:169 size:1073863032 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 - volume id:170 size:1075747088 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579207 - volume id:171 size:1081881400 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596486 - volume id:172 size:1074787304 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 - volume id:174 size:1073824160 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:175 size:1077742472 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559516 - volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 - volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 - volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45574435 replica_placement:100 version:3 modified_at_second:1615580308 - volume id:180 size:1078438448 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773243 - volume id:181 size:1074571672 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745117 - volume id:183 size:1076361616 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572985 - volume id:184 size:1074594216 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 - volume id:185 size:1074099592 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506835 - volume id:186 size:1074220664 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614934 - volume id:187 size:1074396112 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590491 - volume id:188 size:1074732632 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 - volume id:189 size:1074847824 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 - volume id:190 size:1074008968 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447162 - volume id:191 size:1075492960 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357457 - volume id:192 size:1075857384 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 - volume id:193 size:1076616760 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507877 - volume id:194 size:1073985792 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559502 - volume id:195 size:1074158304 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 - volume id:196 size:1074594640 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501690 - volume id:198 size:1075104624 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623371 - volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:200 size:1075630464 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 - volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:204 size:1079766904 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565702 - volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 - volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 - volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 - volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 - volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 - volume id:221 size:1106545536 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 - volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434685 - volume id:226 size:1103504792 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:227 size:1106699864 collection:"collection1" file_count:2827 delete_count:19 deleted_byte_count:5393310 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 - volume id:229 size:1109855256 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:231 size:1112917664 collection:"collection1" file_count:3151 delete_count:19 deleted_byte_count:2852517 replica_placement:100 version:3 modified_at_second:1615611350 - volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611352 - volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:236 size:1089476200 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 - volume id:240 size:424791528 collection:"collection1" file_count:734 delete_count:12 deleted_byte_count:7353071 replica_placement:100 version:3 modified_at_second:1615631669 - volume id:242 size:1075383304 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852115 - volume id:243 size:1088174560 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852202 - volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 - volume id:247 size:1075859784 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 - volume id:249 size:1074819168 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856231 - volume id:250 size:1080572256 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 - volume id:251 size:1075684408 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 - volume id:254 size:1074830800 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 - volume id:257 size:1082621664 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866395 - volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 - volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:262 size:1078492464 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 - volume id:263 size:1077167440 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 - volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877435 - volume id:269 size:1077552720 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 - volume id:271 size:1076992648 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 - volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 - volume id:274 size:1075994128 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 - volume id:276 size:1076899888 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 - volume id:277 size:1074956160 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 - volume id:279 size:1077325096 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890230 - volume id:282 size:1075232240 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 - volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 - volume id:285 size:1082128576 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 - volume id:286 size:1077464824 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 - volume id:287 size:1074590528 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904874 - volume id:288 size:1075406800 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 - volume id:292 size:1092010744 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912327 - volume id:293 size:1075409776 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 - volume id:294 size:1075443992 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 - volume id:295 size:1074702376 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 - volume id:300 size:1076212424 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 - volume id:304 size:1081038888 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925567 - volume id:305 size:1074185552 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 - volume id:310 size:1074761520 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 - volume id:311 size:1088248208 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931460 - volume id:312 size:1075037808 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937832 - volume id:313 size:1074876016 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 - volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 - volume id:315 size:1084153456 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937884 - volume id:316 size:1077720784 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 - volume id:317 size:1076215040 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 - volume id:319 size:1073952744 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 - volume id:320 size:1082437736 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943591 - volume id:321 size:1081477960 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 - volume id:322 size:1078471600 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950574 - volume id:324 size:1075606712 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 - volume id:326 size:1076059936 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 - volume id:327 size:1076121224 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 - volume id:328 size:1074767928 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 - volume id:329 size:1076691776 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 - volume id:331 size:1074957192 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 - volume id:333 size:1074270192 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 - volume id:335 size:1076235176 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 - volume id:336 size:1087853032 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963003 - volume id:338 size:1076118360 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 - volume id:342 size:1080186296 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977518 - volume id:343 size:1075345184 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 - volume id:349 size:1075957824 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 - volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 - volume id:354 size:1085213992 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992207 - volume id:356 size:1083304992 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992027 - volume id:358 size:1085152312 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 - volume id:359 size:1074211240 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 - volume id:362 size:1074074120 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 - volume id:363 size:1078859496 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004822 - volume id:364 size:1081280816 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 - volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 - volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 - volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 - volume id:368 size:1074822016 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010878 - volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011019 - volume id:370 size:1076040480 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:371 size:1078806160 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010994 - volume id:372 size:1076193312 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 - volume id:374 size:1085011080 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 - volume id:375 size:1076140688 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 - volume id:378 size:959273824 collection:"collection2" file_count:15031 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:379 size:1014108592 collection:"collection2" file_count:15360 delete_count:8 deleted_byte_count:2524591 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:380 size:1010760464 collection:"collection2" file_count:14920 delete_count:7 deleted_byte_count:134859 replica_placement:100 version:3 modified_at_second:1615632322 - Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0) - Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0) - DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0) - Disk hdd(volume:299/3000 active:299 free:2701 remote:0) - volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 - volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1615629606 - volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1615631673 - volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1615630117 - volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629340 - volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170024 - volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 - volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 - volume id:29 size:281582688 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 - volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 - volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 - volume id:32 size:281343360 file_count:1497 replica_placement:100 version:3 modified_at_second:1615632362 - volume id:33 size:1130226344 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 - volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 - volume id:35 size:1075241744 collection:"collection1" file_count:10523 delete_count:97 deleted_byte_count:46586217 replica_placement:100 version:3 modified_at_second:1615618790 - volume id:36 size:1075118336 collection:"collection1" file_count:10341 delete_count:118 deleted_byte_count:24753278 replica_placement:100 version:3 modified_at_second:1615606148 - volume id:37 size:1075895576 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594776 - volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 - volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 - volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 - volume id:41 size:1076283592 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19311237 replica_placement:100 version:3 modified_at_second:1615596736 - volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548906 - volume id:43 size:1116445864 collection:"collection1" file_count:7355 delete_count:57 deleted_byte_count:9727158 replica_placement:100 version:3 modified_at_second:1615566167 - volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573498 - volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569825 - volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 - volume id:48 size:1076778664 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 - volume id:49 size:1078775288 collection:"collection1" file_count:9631 delete_count:27 deleted_byte_count:5985628 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615575823 - volume id:50 size:1076688288 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610876 - volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 - volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:54 size:1045022288 collection:"collection1" file_count:9409 delete_count:29 deleted_byte_count:15102818 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630813 - volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631453 - volume id:56 size:1002412240 collection:"collection1" file_count:8762 delete_count:40 deleted_byte_count:65885831 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 - volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631775 - volume id:58 size:908064192 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632343 - volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 - volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 - volume id:61 size:975604544 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 - volume id:62 size:873845904 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 - volume id:63 size:956941176 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 - volume id:64 size:965638424 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631032 - volume id:65 size:823283608 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 - volume id:67 size:878713880 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:69 size:863913896 collection:"collection1" file_count:7291 delete_count:100 deleted_byte_count:25335024 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 - volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 - volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630985 - volume id:73 size:929047544 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 - volume id:75 size:908045000 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 - volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632014 - volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 - volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 - volume id:80 size:827912928 collection:"collection1" file_count:6916 delete_count:15 deleted_byte_count:5611604 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631159 - volume id:81 size:880693168 collection:"collection1" file_count:7481 delete_count:238 deleted_byte_count:80880878 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631395 - volume id:82 size:1041660512 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 - volume id:83 size:936194288 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 - volume id:84 size:871262320 collection:"collection1" file_count:8190 delete_count:14 deleted_byte_count:3150948 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 - volume id:86 size:1009434632 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:87 size:922274624 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 - volume id:88 size:1073767976 collection:"collection1" file_count:14994 delete_count:207 deleted_byte_count:82380696 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615541383 - volume id:89 size:1044421824 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 - volume id:90 size:891163760 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629392 - volume id:91 size:936573952 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 - volume id:92 size:992439144 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 - volume id:93 size:1079602592 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556013 - volume id:94 size:1030684704 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 - volume id:95 size:990877824 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631867 - volume id:96 size:989294848 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 - volume id:98 size:1077836472 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 - volume id:99 size:1071718496 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 - volume id:100 size:1083617472 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505914 - volume id:101 size:1077109408 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864780 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 - volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626682 - volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 - volume id:106 size:1075458680 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 - volume id:107 size:1073811776 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747428 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 - volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593231 - volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 - volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 - volume id:111 size:1073826176 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 - volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 - volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 - volume id:114 size:1074762792 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 - volume id:115 size:1076192296 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285296 - volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573712 - volume id:118 size:1074064344 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516264 - volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 - volume id:121 size:1078803320 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506626 - volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585912 - volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:126 size:1073867408 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585911 - volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612990 - volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 - volume id:129 size:1074704328 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 - volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582327 - volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 - volume id:132 size:1088539552 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619581 - volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555873 - volume id:135 size:1073906776 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 - volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 - volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 - volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 - volume id:142 size:1075733064 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598913 - volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488584 - volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 - volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 - volume id:146 size:1076337576 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 - volume id:147 size:1077130576 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604115 - volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586390 - volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494494 - volume id:150 size:1076290328 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611567 - volume id:152 size:1075941400 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:154 size:1081244696 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478469 - volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:10076804 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628341 - volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 - volume id:159 size:1074132368 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 - volume id:160 size:1075680952 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597780 - volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478126 - volume id:163 size:1074457192 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 - volume id:164 size:1074261248 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620084 - volume id:165 size:1073891016 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471907 - volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491019 - volume id:168 size:1074718808 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 - volume id:169 size:1073863128 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 - volume id:170 size:1075747096 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579204 - volume id:171 size:1081881312 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596485 - volume id:172 size:1074787312 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 - volume id:173 size:1074154648 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2430948 replica_placement:100 version:3 modified_at_second:1615591904 - volume id:175 size:1077742504 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559515 - volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 - volume id:177 size:1074120120 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3719562 replica_placement:100 version:3 modified_at_second:1615516891 - volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 - volume id:180 size:1078438536 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773242 - volume id:181 size:1074571768 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745116 - volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1416142 replica_placement:100 version:3 modified_at_second:1615568922 - volume id:183 size:1076361448 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572982 - volume id:184 size:1074594160 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 - volume id:185 size:1074099624 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506832 - volume id:186 size:1074220864 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614933 - volume id:187 size:1074395944 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590490 - volume id:188 size:1074732792 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 - volume id:189 size:1074847896 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 - volume id:190 size:1074008912 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447161 - volume id:191 size:1075493024 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357456 - volume id:192 size:1075857400 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 - volume id:193 size:1076616768 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507875 - volume id:194 size:1073985624 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559499 - volume id:195 size:1074158312 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 - volume id:196 size:1074594784 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501688 - volume id:197 size:1075423296 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485253 - volume id:198 size:1075104712 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623369 - volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585984 - volume id:200 size:1075630536 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 - volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 - volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 - volume id:204 size:1079766872 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565701 - volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 - volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630989 - volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 - volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:212 size:1078293448 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:213 size:1093588072 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 - volume id:215 size:1074798704 collection:"collection1" file_count:2819 delete_count:31 deleted_byte_count:10873569 replica_placement:100 version:3 modified_at_second:1615586386 - volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 - volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 - volume id:219 size:1092298816 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 - volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 - volume id:221 size:1106545456 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 - volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 - volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 - volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434684 - volume id:226 size:1103504768 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601867 - volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 - volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 - volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:236 size:1089476136 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 - volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 - volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630841 - volume id:240 size:424791528 collection:"collection1" file_count:733 delete_count:13 deleted_byte_count:7515220 replica_placement:100 version:3 modified_at_second:1615631670 - volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 - volume id:242 size:1075383392 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852116 - volume id:243 size:1088174704 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852203 - volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1615628825 - volume id:246 size:1075998672 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852106 - volume id:247 size:1075859808 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 - volume id:248 size:1084301208 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 - volume id:250 size:1080572168 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 - volume id:252 size:1075065264 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861200 - volume id:253 size:1087328880 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861258 - volume id:254 size:1074830736 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 - volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 - volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 - volume id:257 size:1082621720 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866402 - volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:259 size:1075507848 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 - volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 - volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:262 size:1078492584 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 - volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 - volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1614871543 - volume id:266 size:1075617552 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 - volume id:267 size:1075699376 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 - volume id:270 size:1076876424 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 - volume id:271 size:1076992704 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 - volume id:272 size:1076145912 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 - volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 - volume id:274 size:1075994184 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 - volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 - volume id:276 size:1076899880 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 - volume id:278 size:1078798632 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 - volume id:280 size:1077432160 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 - volume id:281 size:1077581064 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 - volume id:282 size:1075232184 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 - volume id:283 size:1080178880 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896623 - volume id:286 size:1077464816 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 - volume id:287 size:1074590536 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904875 - volume id:288 size:1075406920 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 - volume id:289 size:1075284312 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 - volume id:290 size:1074723800 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:292 size:1092010672 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912325 - volume id:295 size:1074702320 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 - volume id:296 size:1077824056 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 - volume id:297 size:1080229176 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 - volume id:298 size:1075410024 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 - volume id:302 size:1077559640 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 - volume id:304 size:1081038944 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925569 - volume id:305 size:1074185376 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 - volume id:306 size:1074763952 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 - volume id:307 size:1076567912 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 - volume id:308 size:1074022336 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 - volume id:311 size:1088248344 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931463 - volume id:312 size:1075037528 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937831 - volume id:313 size:1074875960 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 - volume id:316 size:1077720776 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 - volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 - volume id:322 size:1078471536 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950572 - volume id:323 size:1074608056 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 - volume id:325 size:1080701232 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 - volume id:326 size:1076059920 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 - volume id:327 size:1076121304 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 - volume id:328 size:1074767816 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 - volume id:329 size:1076691960 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 - volume id:330 size:1080825760 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956476 - volume id:331 size:1074957256 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 - volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 - volume id:333 size:1074270160 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 - volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 - volume id:335 size:1076235136 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 - volume id:337 size:1075646896 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 - volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969995 - volume id:340 size:1079462152 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969977 - volume id:341 size:1074448360 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 - volume id:343 size:1075345072 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 - volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1614977511 - volume id:347 size:1075145248 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 - volume id:348 size:1080623544 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984604 - volume id:349 size:1075957672 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 - volume id:351 size:1078795120 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 - volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 - volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 - volume id:354 size:1085214104 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992208 - volume id:357 size:1074276152 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 - volume id:359 size:1074211296 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 - volume id:360 size:1075532512 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 - volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 - volume id:362 size:1074074176 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 - volume id:363 size:1078859640 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004823 - volume id:364 size:1081280880 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 - volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 - volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 - volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 - volume id:368 size:1074821960 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011021 - volume id:370 size:1076040544 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:371 size:1078806216 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010995 - volume id:372 size:1076193344 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 - volume id:373 size:1080928088 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016878 - volume id:374 size:1085011176 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 - volume id:376 size:1074845832 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 - volume id:377 size:957434264 collection:"collection2" file_count:14929 delete_count:1 deleted_byte_count:43099 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:379 size:1014108528 collection:"collection2" file_count:15362 delete_count:6 deleted_byte_count:2481613 replica_placement:100 version:3 modified_at_second:1615632323 - Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 -total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 -` +//go:embed sample.topo.txt +var topoData string diff --git a/weed/shell/command_volume_mark.go b/weed/shell/command_volume_mark.go index 19b614310..53c2265b0 100644 --- a/weed/shell/command_volume_mark.go +++ b/weed/shell/command_volume_mark.go @@ -3,6 +3,7 @@ package shell import ( "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -28,10 +29,6 @@ func (c *commandVolumeMark) Help() string { func (c *commandVolumeMark) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volMarkCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volMarkCommand.Int("volumeId", 0, "the volume id") nodeStr := volMarkCommand.String("node", "", "the volume server :") @@ -47,7 +44,11 @@ func (c *commandVolumeMark) Do(args []string, commandEnv *CommandEnv, writer io. markWritable = true } - sourceVolumeServer := *nodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer := pb.ServerAddress(*nodeStr) volumeId := needle.VolumeId(*volumeIdInt) diff --git a/weed/shell/command_volume_mount.go b/weed/shell/command_volume_mount.go index bd588d0b5..7b03b8dfa 100644 --- a/weed/shell/command_volume_mount.go +++ b/weed/shell/command_volume_mount.go @@ -3,6 +3,7 @@ package shell import ( "context" "flag" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/operation" @@ -34,10 +35,6 @@ func (c *commandVolumeMount) Help() string { func (c *commandVolumeMount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volMountCommand.Int("volumeId", 0, "the volume id") nodeStr := volMountCommand.String("node", "", "the volume server :") @@ -45,7 +42,11 @@ func (c *commandVolumeMount) Do(args []string, commandEnv *CommandEnv, writer io return nil } - sourceVolumeServer := *nodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer := pb.ServerAddress(*nodeStr) volumeId := needle.VolumeId(*volumeIdInt) @@ -53,8 +54,8 @@ func (c *commandVolumeMount) Do(args []string, commandEnv *CommandEnv, writer io } -func mountVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string) (err error) { - return operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { +func mountVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress) (err error) { + return operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{ VolumeId: uint32(volumeId), }) diff --git a/weed/shell/command_volume_move.go b/weed/shell/command_volume_move.go index 3dcf41354..d7cc8b8ce 100644 --- a/weed/shell/command_volume_move.go +++ b/weed/shell/command_volume_move.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/wdclient" "io" "log" @@ -49,10 +50,6 @@ func (c *commandVolumeMove) Help() string { func (c *commandVolumeMove) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volMoveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volMoveCommand.Int("volumeId", 0, "the volume id") sourceNodeStr := volMoveCommand.String("source", "", "the source volume server :") @@ -62,7 +59,11 @@ func (c *commandVolumeMove) Do(args []string, commandEnv *CommandEnv, writer io. return nil } - sourceVolumeServer, targetVolumeServer := *sourceNodeStr, *targetNodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer, targetVolumeServer := pb.ServerAddress(*sourceNodeStr), pb.ServerAddress(*targetNodeStr) volumeId := needle.VolumeId(*volumeIdInt) @@ -74,10 +75,10 @@ func (c *commandVolumeMove) Do(args []string, commandEnv *CommandEnv, writer io. } // LiveMoveVolume moves one volume from one source volume server to one target volume server, with idleTimeout to drain the incoming requests. -func LiveMoveVolume(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer string, idleTimeout time.Duration, diskType string, skipTailError bool) (err error) { +func LiveMoveVolume(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer pb.ServerAddress, idleTimeout time.Duration, diskType string, skipTailError bool) (err error) { log.Printf("copying volume %d from %s to %s", volumeId, sourceVolumeServer, targetVolumeServer) - lastAppendAtNs, err := copyVolume(grpcDialOption, volumeId, sourceVolumeServer, targetVolumeServer, diskType) + lastAppendAtNs, err := copyVolume(grpcDialOption, writer, volumeId, sourceVolumeServer, targetVolumeServer, diskType) if err != nil { return fmt.Errorf("copy volume %d from %s to %s: %v", volumeId, sourceVolumeServer, targetVolumeServer, err) } @@ -100,7 +101,7 @@ func LiveMoveVolume(grpcDialOption grpc.DialOption, writer io.Writer, volumeId n return nil } -func copyVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer string, diskType string) (lastAppendAtNs uint64, err error) { +func copyVolume(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer pb.ServerAddress, diskType string) (lastAppendAtNs uint64, err error) { // check to see if the volume is already read-only and if its not then we need // to mark it as read-only and then before we return we need to undo what we @@ -111,7 +112,7 @@ func copyVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, source return } - clientErr := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + clientErr := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, writableErr := volumeServerClient.VolumeMarkWritable(context.Background(), &volume_server_pb.VolumeMarkWritableRequest{ VolumeId: uint32(volumeId), }) @@ -122,7 +123,7 @@ func copyVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, source } }() - err = operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, statusErr := volumeServerClient.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{ VolumeId: uint32(volumeId), }) @@ -139,37 +140,53 @@ func copyVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, source return } - err = operation.WithVolumeServerClient(targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - resp, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{ + err = operation.WithVolumeServerClient(true, targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + stream, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{ VolumeId: uint32(volumeId), - SourceDataNode: sourceVolumeServer, + SourceDataNode: string(sourceVolumeServer), DiskType: diskType, }) - if replicateErr == nil { - lastAppendAtNs = resp.LastAppendAtNs + if replicateErr != nil { + return replicateErr } - return replicateErr + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + return recvErr + } + } + if resp.LastAppendAtNs != 0 { + lastAppendAtNs = resp.LastAppendAtNs + } else { + fmt.Fprintf(writer, "volume %d processed %d bytes\n", volumeId, resp.ProcessedBytes) + } + } + + return nil }) return } -func tailVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer string, lastAppendAtNs uint64, idleTimeout time.Duration) (err error) { +func tailVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer, targetVolumeServer pb.ServerAddress, lastAppendAtNs uint64, idleTimeout time.Duration) (err error) { - return operation.WithVolumeServerClient(targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(true, targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, replicateErr := volumeServerClient.VolumeTailReceiver(context.Background(), &volume_server_pb.VolumeTailReceiverRequest{ VolumeId: uint32(volumeId), SinceNs: lastAppendAtNs, IdleTimeoutSeconds: uint32(idleTimeout.Seconds()), - SourceVolumeServer: sourceVolumeServer, + SourceVolumeServer: string(sourceVolumeServer), }) return replicateErr }) } -func deleteVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string) (err error) { - return operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { +func deleteVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress) (err error) { + return operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, deleteErr := volumeServerClient.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ VolumeId: uint32(volumeId), }) @@ -177,8 +194,8 @@ func deleteVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sour }) } -func markVolumeWritable(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string, writable bool) (err error) { - return operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { +func markVolumeWritable(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress, writable bool) (err error) { + return operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { if writable { _, err = volumeServerClient.VolumeMarkWritable(context.Background(), &volume_server_pb.VolumeMarkWritableRequest{ VolumeId: uint32(volumeId), @@ -195,7 +212,7 @@ func markVolumeWritable(grpcDialOption grpc.DialOption, volumeId needle.VolumeId func markVolumeReplicasWritable(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, locations []wdclient.Location, writable bool) error { for _, location := range locations { fmt.Printf("markVolumeReadonly %d on %s ...\n", volumeId, location.Url) - if err := markVolumeWritable(grpcDialOption, volumeId, location.Url, writable); err != nil { + if err := markVolumeWritable(grpcDialOption, volumeId, location.ServerAddress(), writable); err != nil { return err } } diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index 6e0c19ae1..4adce095d 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -8,9 +8,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "os" - "sort" ) func init() { @@ -44,10 +44,6 @@ func (c *commandVolumeServerEvacuate) Help() string { func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeServer := vsEvacuateCommand.String("node", "", ": of the volume server") skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved") @@ -56,6 +52,11 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv, if err = vsEvacuateCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyChange, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } if *volumeServer == "" { return fmt.Errorf("need to specify volume server by -node=:") @@ -78,7 +79,7 @@ func volumeServerEvacuate(commandEnv *CommandEnv, volumeServer string, skipNonMo // list all the volumes // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -153,11 +154,9 @@ func evacuateEcVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyI func moveAwayOneEcVolume(commandEnv *CommandEnv, ecShardInfo *master_pb.VolumeEcShardInformationMessage, thisNode *EcNode, otherNodes []*EcNode, applyChange bool) (hasMoved bool, err error) { for _, shardId := range erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds() { - - sort.Slice(otherNodes, func(i, j int) bool { - return otherNodes[i].localShardIdCount(ecShardInfo.Id) < otherNodes[j].localShardIdCount(ecShardInfo.Id) + slices.SortFunc(otherNodes, func(a, b *EcNode) bool { + return a.localShardIdCount(ecShardInfo.Id) < b.localShardIdCount(ecShardInfo.Id) }) - for i := 0; i < len(otherNodes); i++ { emptyNode := otherNodes[i] collectionPrefix := "" @@ -188,10 +187,9 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][ return v.DiskType == vol.DiskType }) } - sort.Slice(otherNodes, func(i, j int) bool { - return otherNodes[i].localVolumeRatio(fn) > otherNodes[j].localVolumeRatio(fn) + slices.SortFunc(otherNodes, func(a, b *Node) bool { + return a.localVolumeRatio(fn) > b.localVolumeRatio(fn) }) - for i := 0; i < len(otherNodes); i++ { emptyNode := otherNodes[i] hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange) diff --git a/weed/shell/command_volume_server_leave.go b/weed/shell/command_volume_server_leave.go index 2a2e56e86..029ef2201 100644 --- a/weed/shell/command_volume_server_leave.go +++ b/weed/shell/command_volume_server_leave.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "google.golang.org/grpc" "io" @@ -36,26 +37,26 @@ func (c *commandVolumeServerLeave) Help() string { func (c *commandVolumeServerLeave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - vsLeaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeServer := vsLeaveCommand.String("node", "", ": of the volume server") if err = vsLeaveCommand.Parse(args); err != nil { return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + if *volumeServer == "" { return fmt.Errorf("need to specify volume server by -node=:") } - return volumeServerLeave(commandEnv.option.GrpcDialOption, *volumeServer, writer) + return volumeServerLeave(commandEnv.option.GrpcDialOption, pb.ServerAddress(*volumeServer), writer) } -func volumeServerLeave(grpcDialOption grpc.DialOption, volumeServer string, writer io.Writer) (err error) { - return operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { +func volumeServerLeave(grpcDialOption grpc.DialOption, volumeServer pb.ServerAddress, writer io.Writer) (err error) { + return operation.WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, leaveErr := volumeServerClient.VolumeServerLeave(context.Background(), &volume_server_pb.VolumeServerLeaveRequest{}) if leaveErr != nil { fmt.Fprintf(writer, "ask volume server %s to leave: %v\n", volumeServer, leaveErr) diff --git a/weed/shell/command_volume_tier_download.go b/weed/shell/command_volume_tier_download.go index 33166ce65..424a64b09 100644 --- a/weed/shell/command_volume_tier_download.go +++ b/weed/shell/command_volume_tier_download.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "google.golang.org/grpc" @@ -42,10 +43,6 @@ func (c *commandVolumeTierDownload) Help() string { func (c *commandVolumeTierDownload) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - tierCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeId := tierCommand.Int("volumeId", 0, "the volume id") collection := tierCommand.String("collection", "", "the collection name") @@ -53,10 +50,14 @@ func (c *commandVolumeTierDownload) Do(args []string, commandEnv *CommandEnv, wr return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + vid := needle.VolumeId(*volumeId) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -112,7 +113,7 @@ func doVolumeTierDownload(commandEnv *CommandEnv, writer io.Writer, collection s // TODO parallelize this for _, loc := range locations { // copy the .dat file from remote tier to local - err = downloadDatFromRemoteTier(commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, loc.Url) + err = downloadDatFromRemoteTier(commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, loc.ServerAddress()) if err != nil { return fmt.Errorf("download dat file for volume %d to %s: %v", vid, loc.Url, err) } @@ -121,9 +122,9 @@ func doVolumeTierDownload(commandEnv *CommandEnv, writer io.Writer, collection s return nil } -func downloadDatFromRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, targetVolumeServer string) error { +func downloadDatFromRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, targetVolumeServer pb.ServerAddress) error { - err := operation.WithVolumeServerClient(targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(true, targetVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { stream, downloadErr := volumeServerClient.VolumeTierMoveDatFromRemote(context.Background(), &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{ VolumeId: uint32(volumeId), Collection: collection, diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index c72958259..4528ca98d 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/wdclient" @@ -19,10 +20,17 @@ func init() { Commands = append(Commands, &commandVolumeTierMove{}) } +type volumeTierMoveJob struct { + src pb.ServerAddress + vid needle.VolumeId +} + type commandVolumeTierMove struct { - activeServers map[string]struct{} - activeServersLock sync.Mutex - activeServersCond *sync.Cond + activeServers sync.Map + queues map[pb.ServerAddress]chan volumeTierMoveJob + //activeServers map[pb.ServerAddress]struct{} + //activeServersLock sync.Mutex + //activeServersCond *sync.Cond } func (c *commandVolumeTierMove) Name() string { @@ -32,7 +40,7 @@ func (c *commandVolumeTierMove) Name() string { func (c *commandVolumeTierMove) Help() string { return `change a volume from one disk type to another - volume.tier.move -fromDiskType=hdd -toDiskType=ssd [-collectionPattern=""] [-fullPercent=95] [-quietFor=1h] + volume.tier.move -fromDiskType=hdd -toDiskType=ssd [-collectionPattern=""] [-fullPercent=95] [-quietFor=1h] [-parallelLimit=4] Even if the volume is replicated, only one replica will be changed and the rest replicas will be dropped. So "volume.fix.replication" and "volume.balance" should be followed. @@ -42,23 +50,22 @@ func (c *commandVolumeTierMove) Help() string { func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - c.activeServers = make(map[string]struct{}) - c.activeServersCond = sync.NewCond(new(sync.Mutex)) - - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - tierCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) collectionPattern := tierCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") fullPercentage := tierCommand.Float64("fullPercent", 95, "the volume reaches the percentage of max volume size") quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period") source := tierCommand.String("fromDiskType", "", "the source disk type") target := tierCommand.String("toDiskType", "", "the target disk type") + parallelLimit := tierCommand.Int("parallelLimit", 0, "limit the number of parallel copying jobs") applyChange := tierCommand.Bool("force", false, "actually apply the changes") if err = tierCommand.Parse(args); err != nil { return nil } + infoAboutSimulationMode(writer, *applyChange, "-force") + + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } fromDiskType := types.ToDiskType(*source) toDiskType := types.ToDiskType(*target) @@ -68,7 +75,7 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer } // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -81,15 +88,86 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer fmt.Printf("tier move volumes: %v\n", volumeIds) _, allLocations := collectVolumeReplicaLocations(topologyInfo) + allLocations = filterLocationsByDiskType(allLocations, toDiskType) + keepDataNodesSorted(allLocations, toDiskType) + + if len(allLocations) > 0 && *parallelLimit > 0 && *parallelLimit < len(allLocations) { + allLocations = allLocations[:*parallelLimit] + } + + wg := sync.WaitGroup{} + bufferLen := len(allLocations) + c.queues = make(map[pb.ServerAddress]chan volumeTierMoveJob) + + for _, dst := range allLocations { + destServerAddress := pb.NewServerAddressFromDataNode(dst.dataNode) + c.queues[destServerAddress] = make(chan volumeTierMoveJob, bufferLen) + + wg.Add(1) + go func(dst location, jobs <-chan volumeTierMoveJob, applyChanges bool) { + defer wg.Done() + for job := range jobs { + fmt.Fprintf(writer, "moving volume %d from %s to %s with disk type %s ...\n", job.vid, job.src, dst.dataNode.Id, toDiskType.ReadableString()) + + locations, found := commandEnv.MasterClient.GetLocations(uint32(job.vid)) + if !found { + fmt.Printf("volume %d not found", job.vid) + continue + } + + unlock := c.Lock(job.src) + + if applyChanges { + if err := c.doMoveOneVolume(commandEnv, writer, job.vid, toDiskType, locations, job.src, dst); err != nil { + fmt.Fprintf(writer, "move volume %d %s => %s: %v\n", job.vid, job.src, dst.dataNode.Id, err) + } + } + unlock() + } + }(dst, c.queues[destServerAddress], *applyChange) + } + for _, vid := range volumeIds { - if err = c.doVolumeTierMove(commandEnv, writer, vid, toDiskType, allLocations, *applyChange); err != nil { + if err = c.doVolumeTierMove(commandEnv, writer, vid, toDiskType, allLocations); err != nil { fmt.Printf("tier move volume %d: %v\n", vid, err) } + allLocations = rotateDataNodes(allLocations) } + for key, _ := range c.queues { + close(c.queues[key]) + } + + wg.Wait() return nil } +func (c *commandVolumeTierMove) Lock(key pb.ServerAddress) func() { + value, _ := c.activeServers.LoadOrStore(key, &sync.Mutex{}) + mtx := value.(*sync.Mutex) + mtx.Lock() + + return func() { mtx.Unlock() } +} + +func filterLocationsByDiskType(dataNodes []location, diskType types.DiskType) (ret []location) { + for _, loc := range dataNodes { + _, found := loc.dataNode.DiskInfos[string(diskType)] + if found { + ret = append(ret, loc) + } + } + return +} + +func rotateDataNodes(dataNodes []location) []location { + if len(dataNodes) > 0 { + return append(dataNodes[1:], dataNodes[0]) + } else { + return dataNodes + } +} + func isOneOf(server string, locations []wdclient.Location) bool { for _, loc := range locations { if server == loc.Url { @@ -99,7 +177,7 @@ func isOneOf(server string, locations []wdclient.Location) bool { return false } -func (c *commandVolumeTierMove) doVolumeTierMove(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, allLocations []location, applyChanges bool) (err error) { +func (c *commandVolumeTierMove) doVolumeTierMove(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, allLocations []location) (err error) { // find volume location locations, found := commandEnv.MasterClient.GetLocations(uint32(vid)) if !found { @@ -108,61 +186,32 @@ func (c *commandVolumeTierMove) doVolumeTierMove(commandEnv *CommandEnv, writer // find one server with the most empty volume slots with target disk type hasFoundTarget := false - keepDataNodesSorted(allLocations, toDiskType) fn := capacityByFreeVolumeCount(toDiskType) - wg := sync.WaitGroup{} for _, dst := range allLocations { if fn(dst.dataNode) > 0 && !hasFoundTarget { // ask the volume server to replicate the volume if isOneOf(dst.dataNode.Id, locations) { continue } - sourceVolumeServer := "" + var sourceVolumeServer pb.ServerAddress for _, loc := range locations { if loc.Url != dst.dataNode.Id { - sourceVolumeServer = loc.Url + sourceVolumeServer = loc.ServerAddress() } } if sourceVolumeServer == "" { continue } - fmt.Fprintf(writer, "moving volume %d from %s to %s with disk type %s ...\n", vid, sourceVolumeServer, dst.dataNode.Id, toDiskType.ReadableString()) hasFoundTarget = true - if !applyChanges { - // adjust volume count - dst.dataNode.DiskInfos[string(toDiskType)].VolumeCount++ - break - } - - c.activeServersCond.L.Lock() - _, isSourceActive := c.activeServers[sourceVolumeServer] - _, isDestActive := c.activeServers[dst.dataNode.Id] - for isSourceActive || isDestActive { - c.activeServersCond.Wait() - _, isSourceActive = c.activeServers[sourceVolumeServer] - _, isDestActive = c.activeServers[dst.dataNode.Id] - } - c.activeServers[sourceVolumeServer] = struct{}{} - c.activeServers[dst.dataNode.Id] = struct{}{} - c.activeServersCond.L.Unlock() - - wg.Add(1) - go func(dst location) { - if err := c.doMoveOneVolume(commandEnv, writer, vid, toDiskType, locations, sourceVolumeServer, dst); err != nil { - fmt.Fprintf(writer, "move volume %d %s => %s: %v\n", vid, sourceVolumeServer, dst.dataNode.Id, err) - } - delete(c.activeServers, sourceVolumeServer) - delete(c.activeServers, dst.dataNode.Id) - c.activeServersCond.Signal() - wg.Done() - }(dst) + // adjust volume count + dst.dataNode.DiskInfos[string(toDiskType)].VolumeCount++ + destServerAddress := pb.NewServerAddressFromDataNode(dst.dataNode) + c.queues[destServerAddress] <- volumeTierMoveJob{sourceVolumeServer, vid} } } - wg.Wait() - if !hasFoundTarget { fmt.Fprintf(writer, "can not find disk type %s for volume %d\n", toDiskType.ReadableString(), vid) } @@ -170,13 +219,13 @@ func (c *commandVolumeTierMove) doVolumeTierMove(commandEnv *CommandEnv, writer return nil } -func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, locations []wdclient.Location, sourceVolumeServer string, dst location) (err error) { +func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, locations []wdclient.Location, sourceVolumeServer pb.ServerAddress, dst location) (err error) { // mark all replicas as read only if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, false); err != nil { return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err) } - if err = LiveMoveVolume(commandEnv.option.GrpcDialOption, writer, vid, sourceVolumeServer, dst.dataNode.Id, 5*time.Second, toDiskType.ReadableString(), true); err != nil { + if err = LiveMoveVolume(commandEnv.option.GrpcDialOption, writer, vid, sourceVolumeServer, pb.NewServerAddressFromDataNode(dst.dataNode), 5*time.Second, toDiskType.ReadableString(), true); err != nil { // mark all replicas as writable if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, true); err != nil { @@ -186,15 +235,13 @@ func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer i return fmt.Errorf("move volume %d %s => %s : %v", vid, locations[0].Url, dst.dataNode.Id, err) } - // adjust volume count - dst.dataNode.DiskInfos[string(toDiskType)].VolumeCount++ - // remove the remaining replicas for _, loc := range locations { - if loc.Url != dst.dataNode.Id && loc.Url != sourceVolumeServer { - if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.Url); err != nil { + if loc.Url != dst.dataNode.Id && loc.ServerAddress() != sourceVolumeServer { + if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress()); err != nil { fmt.Fprintf(writer, "failed to delete volume %d on %s: %v\n", vid, loc.Url, err) } + // reduce volume count? Not really necessary since they are "more" full and will not be a candidate to move to } } return nil diff --git a/weed/shell/command_volume_tier_upload.go b/weed/shell/command_volume_tier_upload.go index c8540a7dc..0df0790e6 100644 --- a/weed/shell/command_volume_tier_upload.go +++ b/weed/shell/command_volume_tier_upload.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "time" @@ -56,10 +57,6 @@ func (c *commandVolumeTierUpload) Help() string { func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - tierCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeId := tierCommand.Int("volumeId", 0, "the volume id") collection := tierCommand.String("collection", "", "the collection name") @@ -71,6 +68,10 @@ func (c *commandVolumeTierUpload) Do(args []string, commandEnv *CommandEnv, writ return nil } + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + vid := needle.VolumeId(*volumeId) // volumeId is provided @@ -107,7 +108,7 @@ func doVolumeTierUpload(commandEnv *CommandEnv, writer io.Writer, collection str } // copy the .dat file to remote tier - err = uploadDatToRemoteTier(commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, locations[0].Url, dest, keepLocalDatFile) + err = uploadDatToRemoteTier(commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, locations[0].ServerAddress(), dest, keepLocalDatFile) if err != nil { return fmt.Errorf("copy dat file for volume %d on %s to %s: %v", vid, locations[0].Url, dest, err) } @@ -115,9 +116,9 @@ func doVolumeTierUpload(commandEnv *CommandEnv, writer io.Writer, collection str return nil } -func uploadDatToRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string, keepLocalDatFile bool) error { +func uploadDatToRemoteTier(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress, dest string, keepLocalDatFile bool) error { - err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(true, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { stream, copyErr := volumeServerClient.VolumeTierMoveDatToRemote(context.Background(), &volume_server_pb.VolumeTierMoveDatToRemoteRequest{ VolumeId: uint32(volumeId), Collection: collection, diff --git a/weed/shell/command_volume_unmount.go b/weed/shell/command_volume_unmount.go index f7e5a501b..85bec44f7 100644 --- a/weed/shell/command_volume_unmount.go +++ b/weed/shell/command_volume_unmount.go @@ -3,6 +3,7 @@ package shell import ( "context" "flag" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "github.com/chrislusf/seaweedfs/weed/operation" @@ -34,10 +35,6 @@ func (c *commandVolumeUnmount) Help() string { func (c *commandVolumeUnmount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volUnmountCommand.Int("volumeId", 0, "the volume id") nodeStr := volUnmountCommand.String("node", "", "the volume server :") @@ -45,7 +42,11 @@ func (c *commandVolumeUnmount) Do(args []string, commandEnv *CommandEnv, writer return nil } - sourceVolumeServer := *nodeStr + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + sourceVolumeServer := pb.ServerAddress(*nodeStr) volumeId := needle.VolumeId(*volumeIdInt) @@ -53,8 +54,8 @@ func (c *commandVolumeUnmount) Do(args []string, commandEnv *CommandEnv, writer } -func unmountVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string) (err error) { - return operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { +func unmountVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress) (err error) { + return operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, unmountErr := volumeServerClient.VolumeUnmount(context.Background(), &volume_server_pb.VolumeUnmountRequest{ VolumeId: uint32(volumeId), }) diff --git a/weed/shell/command_volume_vacuum.go b/weed/shell/command_volume_vacuum.go index 56f85f4fe..37b8aa030 100644 --- a/weed/shell/command_volume_vacuum.go +++ b/weed/shell/command_volume_vacuum.go @@ -22,26 +22,30 @@ func (c *commandVacuum) Name() string { func (c *commandVacuum) Help() string { return `compact volumes if deleted entries are more than the limit - volume.vacuum [-garbageThreshold=0.3] + volume.vacuum [-garbageThreshold=0.3] [-collection=] [-volumeId=] ` } func (c *commandVacuum) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - if err = commandEnv.confirmIsLocked(); err != nil { - return - } - volumeVacuumCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) garbageThreshold := volumeVacuumCommand.Float64("garbageThreshold", 0.3, "vacuum when garbage is more than this limit") + collection := volumeVacuumCommand.String("collection", "", "vacuum this collection") + volumeId := volumeVacuumCommand.Uint("volumeId", 0, "the volume id") if err = volumeVacuumCommand.Parse(args); err != nil { return nil } - err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error { + if err = commandEnv.confirmIsLocked(args); err != nil { + return + } + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err = client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{ GarbageThreshold: float32(*garbageThreshold), + VolumeId: uint32(*volumeId), + Collection: *collection, }) return err }) diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 5497e89cc..08f639d34 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -22,14 +22,15 @@ type ShellOptions struct { // shell transient context FilerHost string FilerPort int64 - FilerAddress string + FilerGroup *string + FilerAddress pb.ServerAddress Directory string } type CommandEnv struct { env map[string]string MasterClient *wdclient.MasterClient - option ShellOptions + option *ShellOptions locker *exclusive_locks.ExclusiveLocker } @@ -43,13 +44,13 @@ var ( Commands = []command{} ) -func NewCommandEnv(options ShellOptions) *CommandEnv { +func NewCommandEnv(options *ShellOptions) *CommandEnv { ce := &CommandEnv{ env: make(map[string]string), - MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(*options.Masters, ",")), + MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, *options.FilerGroup, pb.AdminShellClient, "", "", pb.ServerAddresses(*options.Masters).ToAddressMap()), option: options, } - ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient) + ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient, "admin") return ce } @@ -70,11 +71,12 @@ func (ce *CommandEnv) isDirectory(path string) bool { } -func (ce *CommandEnv) confirmIsLocked() error { +func (ce *CommandEnv) confirmIsLocked(args []string) error { if ce.locker.IsLocking() { return nil } + ce.locker.SetMessage(fmt.Sprintf("%v", args)) return fmt.Errorf("need to run \"lock\" first to continue") @@ -96,10 +98,9 @@ func (ce *CommandEnv) checkDirectory(path string) error { var _ = filer_pb.FilerClient(&CommandEnv{}) -func (ce *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { +func (ce *CommandEnv) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - filerGrpcAddress := fmt.Sprintf("%s:%d", ce.option.FilerHost, ce.option.FilerPort+10000) - return pb.WithGrpcFilerClient(filerGrpcAddress, ce.option.GrpcDialOption, fn) + return pb.WithGrpcFilerClient(streamingMode, ce.option.FilerAddress, ce.option.GrpcDialOption, fn) } diff --git a/weed/shell/sample.topo.txt b/weed/shell/sample.topo.txt new file mode 100644 index 000000000..b3ceaa140 --- /dev/null +++ b/weed/shell/sample.topo.txt @@ -0,0 +1,804 @@ +Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) + DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0) + Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0) + Rack DefaultRack total size:0 file_count:0 + DataCenter dc1 total size:0 file_count:0 + DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0) + Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0) + DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0) + Disk hdd(volume:50/240 active:50 free:190 remote:0) + volume id:15 size:1115965064 collection:"collection0" file_count:83 replica_placement:100 version:3 modified_at_second:1609923671 + volume id:21 size:1097631536 collection:"collection0" file_count:82 delete_count:7 deleted_byte_count:68975485 replica_placement:100 version:3 modified_at_second:1609929578 + volume id:22 size:1086828272 collection:"collection0" file_count:75 replica_placement:100 version:3 modified_at_second:1609930001 + volume id:23 size:1076380216 collection:"collection0" file_count:68 replica_placement:100 version:3 modified_at_second:1609930434 + volume id:24 size:1074139776 collection:"collection0" file_count:90 replica_placement:100 version:3 modified_at_second:1609930909 + volume id:25 size:690757512 collection:"collection0" file_count:38 replica_placement:100 version:3 modified_at_second:1611144216 + volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 + volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 + volume id:29 size:281582680 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 + volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 + volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 + volume id:33 size:1130226400 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 + volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 + volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 + volume id:52 size:1083529728 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 + volume id:54 size:1045022344 collection:"collection1" file_count:9408 delete_count:30 deleted_byte_count:15132106 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:63 size:956941112 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 + volume id:69 size:869213648 collection:"collection1" file_count:7293 delete_count:102 deleted_byte_count:30643207 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 + volume id:74 size:957046128 collection:"collection1" file_count:6982 delete_count:258 deleted_byte_count:73054259 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 + volume id:80 size:827912928 collection:"collection1" file_count:6914 delete_count:17 deleted_byte_count:5689635 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631157 + volume id:84 size:873121856 collection:"collection1" file_count:8200 delete_count:13 deleted_byte_count:3131676 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 + volume id:85 size:1023869320 collection:"collection1" file_count:7788 delete_count:234 deleted_byte_count:78037967 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 + volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631193 + volume id:98 size:1077836440 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 + volume id:105 size:1073996824 collection:"collection1" file_count:6872 delete_count:20 deleted_byte_count:14482293 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499757 + volume id:106 size:1075458664 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 + volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 + volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611565 + volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573878 + volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 + volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584779 + volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 + volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 + volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 + volume id:161 size:1077397192 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 + volume id:173 size:1074154704 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2578509 replica_placement:100 version:3 modified_at_second:1615591904 + volume id:174 size:1073824232 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:197 size:1075423240 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485254 + volume id:219 size:1092298904 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 + volume id:263 size:1077167352 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 + volume id:272 size:1076146040 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 + volume id:291 size:1076256760 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:299 size:1075147824 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918454 + volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 + volume id:302 size:1077559792 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 + volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969996 + volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977458 + volume id:355 size:1075239792 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 + volume id:373 size:1080928000 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016877 + Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0) + DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0) + Disk hdd(volume:36/190 active:36 free:154 remote:0) + volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:480564 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 + volume id:3 size:308743136 file_count:1638 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 + volume id:4 size:285986968 file_count:1641 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 + volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 + volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 + volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 + volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814550 + volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819732 + volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827837 + volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956980 + volume id:16 size:1113666104 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:21 size:1097631664 collection:"collection0" file_count:82 delete_count:11 deleted_byte_count:68985100 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:56 size:1001897616 collection:"collection1" file_count:8762 delete_count:37 deleted_byte_count:65375405 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 + volume id:81 size:880693104 collection:"collection1" file_count:7481 delete_count:236 deleted_byte_count:80386421 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631396 + volume id:104 size:1076383624 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100728071 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602658 + volume id:107 size:1073811840 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747484 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 + volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 + volume id:139 size:1074163936 collection:"collection1" file_count:9315 delete_count:42 deleted_byte_count:10630966 replica_placement:100 version:3 modified_at_second:1615571946 + volume id:151 size:1098659752 collection:"collection1" file_count:10808 delete_count:24 deleted_byte_count:7088102 replica_placement:100 version:3 modified_at_second:1615586389 + volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:9076141 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:167 size:1073958176 collection:"collection1" file_count:25229 delete_count:48 deleted_byte_count:25871565 replica_placement:100 version:3 modified_at_second:1615602669 + volume id:177 size:1074120216 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3803952 replica_placement:100 version:3 modified_at_second:1615516892 + volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45552859 replica_placement:100 version:3 modified_at_second:1615580308 + volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1452346 replica_placement:100 version:3 modified_at_second:1615568922 + volume id:215 size:1068268216 collection:"collection1" file_count:2813 delete_count:10 deleted_byte_count:5676795 replica_placement:100 version:3 modified_at_second:1615586386 + volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 + volume id:283 size:1080178944 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896626 + volume id:303 size:1075944504 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 + volume id:309 size:1075178624 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 + volume id:323 size:1074608200 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 + volume id:344 size:1075035448 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 + volume id:347 size:1075145496 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 + volume id:357 size:1074276208 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 + volume id:380 size:1010760456 collection:"collection2" file_count:14921 delete_count:6 deleted_byte_count:65678 replica_placement:100 version:3 modified_at_second:1615632322 + volume id:381 size:939292792 collection:"collection2" file_count:14619 delete_count:2 deleted_byte_count:5119 replica_placement:100 version:3 modified_at_second:1615632324 + Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 + DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0) + Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0) + DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0) + Disk hdd(volume:108/1850 active:108 free:1742 remote:0) + volume id:1 size:284685936 file_count:1557 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 + volume id:32 size:281390512 file_count:1496 delete_count:6 deleted_byte_count:546403 replica_placement:100 version:3 modified_at_second:1615632362 + volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 + volume id:49 size:1078775288 collection:"collection1" file_count:9636 delete_count:22 deleted_byte_count:5625976 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630446 + volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:88 size:1073767976 collection:"collection1" file_count:14995 delete_count:206 deleted_byte_count:81222360 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629897 + volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 + volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 + volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 + volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615564274 + volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 + volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 + volume id:212 size:1078293360 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:213 size:1093587976 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 + volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 + volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 + volume id:227 size:1106699896 collection:"collection1" file_count:2827 delete_count:20 deleted_byte_count:5496790 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:229 size:1109855312 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609988 + volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 + volume id:231 size:1112917696 collection:"collection1" file_count:3151 delete_count:20 deleted_byte_count:2989828 replica_placement:100 version:3 modified_at_second:1615611350 + volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611352 + volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 + volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630838 + volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 + volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1614852171 + volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 + volume id:246 size:1075998648 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852105 + volume id:248 size:1084301184 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 + volume id:249 size:1074819136 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856230 + volume id:251 size:1075684488 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 + volume id:252 size:1075065208 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861196 + volume id:253 size:1087328816 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861255 + volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 + volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 + volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:259 size:1075507776 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 + volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 + volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1615629130 + volume id:266 size:1075617464 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 + volume id:267 size:1075699544 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 + volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877434 + volume id:269 size:1077552872 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 + volume id:270 size:1076876568 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 + volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 + volume id:277 size:1074956288 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 + volume id:278 size:1078798640 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 + volume id:279 size:1077325040 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890229 + volume id:280 size:1077432216 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 + volume id:281 size:1077581096 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 + volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 + volume id:285 size:1082128688 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 + volume id:289 size:1075284256 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 + volume id:290 size:1074723792 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:291 size:1076256768 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:293 size:1075409792 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 + volume id:294 size:1075444048 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 + volume id:296 size:1077824032 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 + volume id:297 size:1080229136 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 + volume id:298 size:1075410136 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 + volume id:299 size:1075147936 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918455 + volume id:300 size:1076212392 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 + volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 + volume id:303 size:1075944480 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 + volume id:306 size:1074764016 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 + volume id:307 size:1076568000 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 + volume id:308 size:1074022392 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 + volume id:309 size:1075178664 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 + volume id:310 size:1074761528 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 + volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 + volume id:315 size:1084153544 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937885 + volume id:317 size:1076215096 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 + volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 + volume id:319 size:1073952880 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 + volume id:320 size:1082437792 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943599 + volume id:321 size:1081477904 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 + volume id:324 size:1075606680 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 + volume id:325 size:1080701144 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 + volume id:330 size:1080825832 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956477 + volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 + volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 + volume id:336 size:1087853056 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963005 + volume id:337 size:1075646784 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 + volume id:338 size:1076118304 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 + volume id:340 size:1079462184 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969978 + volume id:341 size:1074448400 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 + volume id:342 size:1080186424 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977521 + volume id:344 size:1075035416 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 + volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977457 + volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1615631322 + volume id:348 size:1080623640 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984606 + volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 + volume id:351 size:1078795112 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 + volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 + volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 + volume id:355 size:1075239728 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 + volume id:356 size:1083305048 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992028 + volume id:358 size:1085152368 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 + volume id:360 size:1075532456 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 + volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 + volume id:375 size:1076140568 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 + volume id:376 size:1074845944 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 + volume id:377 size:957284144 collection:"collection2" file_count:14923 delete_count:1 deleted_byte_count:1797 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:378 size:959273936 collection:"collection2" file_count:15027 delete_count:4 deleted_byte_count:231414 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:381 size:939261032 collection:"collection2" file_count:14615 delete_count:5 deleted_byte_count:1192272 replica_placement:100 version:3 modified_at_second:1615632324 + Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0) + Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0) + DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0) + Disk hdd(volume:267/2000 active:267 free:1733 remote:0) + volume id:1 size:284693256 file_count:1558 delete_count:2 deleted_byte_count:4818 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 + volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:464508 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 + volume id:3 size:308741952 file_count:1637 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 + volume id:4 size:285986968 file_count:1640 delete_count:1 deleted_byte_count:145095 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 + volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 + volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 + volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 + volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 + volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814543 + volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1609815123 + volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819726 + volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827832 + volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956983 + volume id:15 size:1115965160 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:4956 replica_placement:100 version:3 modified_at_second:1612957001 + volume id:16 size:1113666048 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1612957011 + volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1612957011 + volume id:22 size:1086828368 collection:"collection0" file_count:75 delete_count:3 deleted_byte_count:5551 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:23 size:1076380280 collection:"collection0" file_count:68 delete_count:2 deleted_byte_count:2910 replica_placement:100 version:3 modified_at_second:1612957011 + volume id:24 size:1074139808 collection:"collection0" file_count:90 delete_count:1 deleted_byte_count:1977 replica_placement:100 version:3 modified_at_second:1612957011 + volume id:25 size:690757544 collection:"collection0" file_count:38 delete_count:1 deleted_byte_count:1944 replica_placement:100 version:3 modified_at_second:1612956995 + volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170023 + volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 + volume id:35 size:1075241656 collection:"collection1" file_count:10523 delete_count:96 deleted_byte_count:46618989 replica_placement:100 version:3 modified_at_second:1615618790 + volume id:36 size:1075118360 collection:"collection1" file_count:10342 delete_count:116 deleted_byte_count:25493106 replica_placement:100 version:3 modified_at_second:1615606148 + volume id:37 size:1075895584 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594777 + volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 + volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 + volume id:41 size:1076283528 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19319268 replica_placement:100 version:3 modified_at_second:1615596736 + volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548908 + volume id:43 size:1116445864 collection:"collection1" file_count:7358 delete_count:54 deleted_byte_count:9534379 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573499 + volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569826 + volume id:48 size:1076778720 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 + volume id:50 size:1076688224 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610879 + volume id:52 size:1083529704 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 + volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631452 + volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631774 + volume id:58 size:908064200 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632342 + volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 + volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 + volume id:61 size:975604488 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 + volume id:62 size:873845936 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 + volume id:64 size:965638488 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631031 + volume id:65 size:823283552 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632306 + volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 + volume id:67 size:878713872 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 + volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630982 + volume id:73 size:929047664 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 + volume id:74 size:957046128 collection:"collection1" file_count:6981 delete_count:259 deleted_byte_count:73080838 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 + volume id:75 size:908044992 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 + volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632013 + volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 + volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 + volume id:82 size:1041661800 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 + volume id:83 size:936195856 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 + volume id:85 size:1023867520 collection:"collection1" file_count:7787 delete_count:240 deleted_byte_count:82091742 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 + volume id:86 size:1009437488 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:87 size:922276640 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 + volume id:89 size:1044401976 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 + volume id:90 size:891145784 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629390 + volume id:91 size:936572832 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 + volume id:92 size:992440712 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 + volume id:93 size:1079603768 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556015 + volume id:94 size:1030685824 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 + volume id:95 size:990879168 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631866 + volume id:96 size:989296136 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 + volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631194 + volume id:99 size:1071718504 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 + volume id:100 size:1083617440 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505917 + volume id:101 size:1077109520 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 + volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626683 + volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 + volume id:104 size:1076383768 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100578087 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602661 + volume id:105 size:1073996824 collection:"collection1" file_count:6873 delete_count:19 deleted_byte_count:14271533 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499756 + volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 + volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 + volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 + volume id:111 size:1073826232 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593233 + volume id:114 size:1074762784 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 + volume id:115 size:1076192240 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285295 + volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611567 + volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573714 + volume id:118 size:1074064400 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516265 + volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573880 + volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 + volume id:121 size:1078803248 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506629 + volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:126 size:1073867464 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585912 + volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612991 + volume id:129 size:1074704272 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 + volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582330 + volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 + volume id:132 size:1088539496 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619584 + volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584780 + volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555876 + volume id:135 size:1073906720 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 + volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:139 size:1074163936 collection:"collection1" file_count:9314 delete_count:43 deleted_byte_count:10631353 replica_placement:100 version:3 modified_at_second:1615571946 + volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 + volume id:142 size:1075732992 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488586 + volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 + volume id:146 size:1076337520 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 + volume id:147 size:1077130544 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604117 + volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586393 + volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494496 + volume id:150 size:1076290408 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611569 + volume id:151 size:1098659752 collection:"collection1" file_count:10805 delete_count:27 deleted_byte_count:7209106 replica_placement:100 version:3 modified_at_second:1615586390 + volume id:152 size:1075941376 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:154 size:1081244752 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478471 + volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628342 + volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 + volume id:159 size:1074132336 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 + volume id:160 size:1075680976 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:161 size:1077397136 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 + volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478127 + volume id:163 size:1074457224 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 + volume id:164 size:1074261256 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620085 + volume id:165 size:1073891080 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471910 + volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491021 + volume id:167 size:1073958280 collection:"collection1" file_count:25231 delete_count:48 deleted_byte_count:26022344 replica_placement:100 version:3 modified_at_second:1615620014 + volume id:168 size:1074718864 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 + volume id:169 size:1073863032 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 + volume id:170 size:1075747088 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579207 + volume id:171 size:1081881400 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596486 + volume id:172 size:1074787304 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 + volume id:174 size:1073824160 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:175 size:1077742472 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559516 + volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 + volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 + volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45574435 replica_placement:100 version:3 modified_at_second:1615580308 + volume id:180 size:1078438448 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773243 + volume id:181 size:1074571672 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745117 + volume id:183 size:1076361616 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572985 + volume id:184 size:1074594216 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 + volume id:185 size:1074099592 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506835 + volume id:186 size:1074220664 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614934 + volume id:187 size:1074396112 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590491 + volume id:188 size:1074732632 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 + volume id:189 size:1074847824 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 + volume id:190 size:1074008968 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447162 + volume id:191 size:1075492960 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357457 + volume id:192 size:1075857384 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 + volume id:193 size:1076616760 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507877 + volume id:194 size:1073985792 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559502 + volume id:195 size:1074158304 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 + volume id:196 size:1074594640 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501690 + volume id:198 size:1075104624 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623371 + volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:200 size:1075630464 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 + volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:204 size:1079766904 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565702 + volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 + volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 + volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 + volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 + volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 + volume id:221 size:1106545536 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 + volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434685 + volume id:226 size:1103504792 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:227 size:1106699864 collection:"collection1" file_count:2827 delete_count:19 deleted_byte_count:5393310 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 + volume id:229 size:1109855256 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:231 size:1112917664 collection:"collection1" file_count:3151 delete_count:19 deleted_byte_count:2852517 replica_placement:100 version:3 modified_at_second:1615611350 + volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611352 + volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:236 size:1089476200 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 + volume id:240 size:424791528 collection:"collection1" file_count:734 delete_count:12 deleted_byte_count:7353071 replica_placement:100 version:3 modified_at_second:1615631669 + volume id:242 size:1075383304 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852115 + volume id:243 size:1088174560 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852202 + volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 + volume id:247 size:1075859784 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 + volume id:249 size:1074819168 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856231 + volume id:250 size:1080572256 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 + volume id:251 size:1075684408 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 + volume id:254 size:1074830800 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 + volume id:257 size:1082621664 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866395 + volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 + volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:262 size:1078492464 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 + volume id:263 size:1077167440 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 + volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877435 + volume id:269 size:1077552720 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 + volume id:271 size:1076992648 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 + volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 + volume id:274 size:1075994128 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 + volume id:276 size:1076899888 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 + volume id:277 size:1074956160 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 + volume id:279 size:1077325096 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890230 + volume id:282 size:1075232240 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 + volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 + volume id:285 size:1082128576 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 + volume id:286 size:1077464824 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 + volume id:287 size:1074590528 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904874 + volume id:288 size:1075406800 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 + volume id:292 size:1092010744 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912327 + volume id:293 size:1075409776 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 + volume id:294 size:1075443992 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 + volume id:295 size:1074702376 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 + volume id:300 size:1076212424 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 + volume id:304 size:1081038888 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925567 + volume id:305 size:1074185552 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 + volume id:310 size:1074761520 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 + volume id:311 size:1088248208 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931460 + volume id:312 size:1075037808 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937832 + volume id:313 size:1074876016 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 + volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 + volume id:315 size:1084153456 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937884 + volume id:316 size:1077720784 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 + volume id:317 size:1076215040 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 + volume id:319 size:1073952744 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 + volume id:320 size:1082437736 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943591 + volume id:321 size:1081477960 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 + volume id:322 size:1078471600 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950574 + volume id:324 size:1075606712 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 + volume id:326 size:1076059936 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 + volume id:327 size:1076121224 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 + volume id:328 size:1074767928 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 + volume id:329 size:1076691776 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 + volume id:331 size:1074957192 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 + volume id:333 size:1074270192 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 + volume id:335 size:1076235176 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 + volume id:336 size:1087853032 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963003 + volume id:338 size:1076118360 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 + volume id:342 size:1080186296 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977518 + volume id:343 size:1075345184 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 + volume id:349 size:1075957824 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 + volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 + volume id:354 size:1085213992 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992207 + volume id:356 size:1083304992 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992027 + volume id:358 size:1085152312 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 + volume id:359 size:1074211240 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 + volume id:362 size:1074074120 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 + volume id:363 size:1078859496 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004822 + volume id:364 size:1081280816 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 + volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 + volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 + volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 + volume id:368 size:1074822016 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010878 + volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011019 + volume id:370 size:1076040480 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:371 size:1078806160 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010994 + volume id:372 size:1076193312 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 + volume id:374 size:1085011080 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 + volume id:375 size:1076140688 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 + volume id:378 size:959273824 collection:"collection2" file_count:15031 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:379 size:1014108592 collection:"collection2" file_count:15360 delete_count:8 deleted_byte_count:2524591 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:380 size:1010760464 collection:"collection2" file_count:14920 delete_count:7 deleted_byte_count:134859 replica_placement:100 version:3 modified_at_second:1615632322 + Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0) + Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0) + DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0) + Disk hdd(volume:299/3000 active:299 free:2701 remote:0) + volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 + volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1615629606 + volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1615631673 + volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1615630117 + volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629340 + volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170024 + volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 + volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 + volume id:29 size:281582688 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 + volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 + volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 + volume id:32 size:281343360 file_count:1497 replica_placement:100 version:3 modified_at_second:1615632362 + volume id:33 size:1130226344 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 + volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 + volume id:35 size:1075241744 collection:"collection1" file_count:10523 delete_count:97 deleted_byte_count:46586217 replica_placement:100 version:3 modified_at_second:1615618790 + volume id:36 size:1075118336 collection:"collection1" file_count:10341 delete_count:118 deleted_byte_count:24753278 replica_placement:100 version:3 modified_at_second:1615606148 + volume id:37 size:1075895576 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594776 + volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 + volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 + volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 + volume id:41 size:1076283592 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19311237 replica_placement:100 version:3 modified_at_second:1615596736 + volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548906 + volume id:43 size:1116445864 collection:"collection1" file_count:7355 delete_count:57 deleted_byte_count:9727158 replica_placement:100 version:3 modified_at_second:1615566167 + volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573498 + volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569825 + volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 + volume id:48 size:1076778664 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 + volume id:49 size:1078775288 collection:"collection1" file_count:9631 delete_count:27 deleted_byte_count:5985628 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615575823 + volume id:50 size:1076688288 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610876 + volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 + volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:54 size:1045022288 collection:"collection1" file_count:9409 delete_count:29 deleted_byte_count:15102818 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630813 + volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631453 + volume id:56 size:1002412240 collection:"collection1" file_count:8762 delete_count:40 deleted_byte_count:65885831 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 + volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631775 + volume id:58 size:908064192 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632343 + volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 + volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 + volume id:61 size:975604544 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 + volume id:62 size:873845904 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 + volume id:63 size:956941176 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 + volume id:64 size:965638424 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631032 + volume id:65 size:823283608 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 + volume id:67 size:878713880 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:69 size:863913896 collection:"collection1" file_count:7291 delete_count:100 deleted_byte_count:25335024 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 + volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 + volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630985 + volume id:73 size:929047544 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 + volume id:75 size:908045000 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 + volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632014 + volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 + volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 + volume id:80 size:827912928 collection:"collection1" file_count:6916 delete_count:15 deleted_byte_count:5611604 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631159 + volume id:81 size:880693168 collection:"collection1" file_count:7481 delete_count:238 deleted_byte_count:80880878 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631395 + volume id:82 size:1041660512 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 + volume id:83 size:936194288 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 + volume id:84 size:871262320 collection:"collection1" file_count:8190 delete_count:14 deleted_byte_count:3150948 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 + volume id:86 size:1009434632 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:87 size:922274624 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 + volume id:88 size:1073767976 collection:"collection1" file_count:14994 delete_count:207 deleted_byte_count:82380696 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615541383 + volume id:89 size:1044421824 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 + volume id:90 size:891163760 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629392 + volume id:91 size:936573952 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 + volume id:92 size:992439144 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 + volume id:93 size:1079602592 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556013 + volume id:94 size:1030684704 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 + volume id:95 size:990877824 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631867 + volume id:96 size:989294848 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 + volume id:98 size:1077836472 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 + volume id:99 size:1071718496 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 + volume id:100 size:1083617472 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505914 + volume id:101 size:1077109408 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864780 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 + volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626682 + volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 + volume id:106 size:1075458680 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 + volume id:107 size:1073811776 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747428 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 + volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593231 + volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 + volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 + volume id:111 size:1073826176 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 + volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 + volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 + volume id:114 size:1074762792 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 + volume id:115 size:1076192296 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285296 + volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573712 + volume id:118 size:1074064344 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516264 + volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 + volume id:121 size:1078803320 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506626 + volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585912 + volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:126 size:1073867408 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585911 + volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612990 + volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 + volume id:129 size:1074704328 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 + volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582327 + volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 + volume id:132 size:1088539552 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619581 + volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555873 + volume id:135 size:1073906776 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 + volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 + volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 + volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 + volume id:142 size:1075733064 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598913 + volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488584 + volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 + volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 + volume id:146 size:1076337576 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 + volume id:147 size:1077130576 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604115 + volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586390 + volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494494 + volume id:150 size:1076290328 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611567 + volume id:152 size:1075941400 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:154 size:1081244696 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478469 + volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:10076804 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628341 + volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 + volume id:159 size:1074132368 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 + volume id:160 size:1075680952 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597780 + volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478126 + volume id:163 size:1074457192 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 + volume id:164 size:1074261248 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620084 + volume id:165 size:1073891016 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471907 + volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491019 + volume id:168 size:1074718808 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 + volume id:169 size:1073863128 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 + volume id:170 size:1075747096 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579204 + volume id:171 size:1081881312 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596485 + volume id:172 size:1074787312 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 + volume id:173 size:1074154648 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2430948 replica_placement:100 version:3 modified_at_second:1615591904 + volume id:175 size:1077742504 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559515 + volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 + volume id:177 size:1074120120 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3719562 replica_placement:100 version:3 modified_at_second:1615516891 + volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 + volume id:180 size:1078438536 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773242 + volume id:181 size:1074571768 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745116 + volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1416142 replica_placement:100 version:3 modified_at_second:1615568922 + volume id:183 size:1076361448 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572982 + volume id:184 size:1074594160 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 + volume id:185 size:1074099624 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506832 + volume id:186 size:1074220864 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614933 + volume id:187 size:1074395944 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590490 + volume id:188 size:1074732792 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 + volume id:189 size:1074847896 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 + volume id:190 size:1074008912 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447161 + volume id:191 size:1075493024 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357456 + volume id:192 size:1075857400 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 + volume id:193 size:1076616768 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507875 + volume id:194 size:1073985624 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559499 + volume id:195 size:1074158312 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 + volume id:196 size:1074594784 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501688 + volume id:197 size:1075423296 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485253 + volume id:198 size:1075104712 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623369 + volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585984 + volume id:200 size:1075630536 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 + volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 + volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 + volume id:204 size:1079766872 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565701 + volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 + volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630989 + volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 + volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:212 size:1078293448 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:213 size:1093588072 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 + volume id:215 size:1074798704 collection:"collection1" file_count:2819 delete_count:31 deleted_byte_count:10873569 replica_placement:100 version:3 modified_at_second:1615586386 + volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 + volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 + volume id:219 size:1092298816 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 + volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 + volume id:221 size:1106545456 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 + volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 + volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 + volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434684 + volume id:226 size:1103504768 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601867 + volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 + volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 + volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:236 size:1089476136 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 + volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 + volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630841 + volume id:240 size:424791528 collection:"collection1" file_count:733 delete_count:13 deleted_byte_count:7515220 replica_placement:100 version:3 modified_at_second:1615631670 + volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 + volume id:242 size:1075383392 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852116 + volume id:243 size:1088174704 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852203 + volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1615628825 + volume id:246 size:1075998672 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852106 + volume id:247 size:1075859808 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 + volume id:248 size:1084301208 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 + volume id:250 size:1080572168 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 + volume id:252 size:1075065264 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861200 + volume id:253 size:1087328880 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861258 + volume id:254 size:1074830736 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 + volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 + volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 + volume id:257 size:1082621720 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866402 + volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:259 size:1075507848 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 + volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 + volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:262 size:1078492584 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 + volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 + volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1614871543 + volume id:266 size:1075617552 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 + volume id:267 size:1075699376 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 + volume id:270 size:1076876424 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 + volume id:271 size:1076992704 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 + volume id:272 size:1076145912 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 + volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 + volume id:274 size:1075994184 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 + volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 + volume id:276 size:1076899880 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 + volume id:278 size:1078798632 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 + volume id:280 size:1077432160 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 + volume id:281 size:1077581064 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 + volume id:282 size:1075232184 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 + volume id:283 size:1080178880 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896623 + volume id:286 size:1077464816 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 + volume id:287 size:1074590536 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904875 + volume id:288 size:1075406920 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 + volume id:289 size:1075284312 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 + volume id:290 size:1074723800 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:292 size:1092010672 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912325 + volume id:295 size:1074702320 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 + volume id:296 size:1077824056 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 + volume id:297 size:1080229176 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 + volume id:298 size:1075410024 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 + volume id:302 size:1077559640 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 + volume id:304 size:1081038944 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925569 + volume id:305 size:1074185376 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 + volume id:306 size:1074763952 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 + volume id:307 size:1076567912 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 + volume id:308 size:1074022336 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 + volume id:311 size:1088248344 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931463 + volume id:312 size:1075037528 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937831 + volume id:313 size:1074875960 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 + volume id:316 size:1077720776 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 + volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 + volume id:322 size:1078471536 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950572 + volume id:323 size:1074608056 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 + volume id:325 size:1080701232 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 + volume id:326 size:1076059920 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 + volume id:327 size:1076121304 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 + volume id:328 size:1074767816 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 + volume id:329 size:1076691960 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 + volume id:330 size:1080825760 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956476 + volume id:331 size:1074957256 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 + volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 + volume id:333 size:1074270160 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 + volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 + volume id:335 size:1076235136 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 + volume id:337 size:1075646896 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 + volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969995 + volume id:340 size:1079462152 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969977 + volume id:341 size:1074448360 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 + volume id:343 size:1075345072 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 + volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1614977511 + volume id:347 size:1075145248 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 + volume id:348 size:1080623544 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984604 + volume id:349 size:1075957672 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 + volume id:351 size:1078795120 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 + volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 + volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 + volume id:354 size:1085214104 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992208 + volume id:357 size:1074276152 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 + volume id:359 size:1074211296 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 + volume id:360 size:1075532512 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 + volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 + volume id:362 size:1074074176 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 + volume id:363 size:1078859640 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004823 + volume id:364 size:1081280880 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 + volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 + volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 + volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 + volume id:368 size:1074821960 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011021 + volume id:370 size:1076040544 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:371 size:1078806216 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010995 + volume id:372 size:1076193344 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 + volume id:373 size:1080928088 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016878 + volume id:374 size:1085011176 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 + volume id:376 size:1074845832 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 + volume id:377 size:957434264 collection:"collection2" file_count:14929 delete_count:1 deleted_byte_count:43099 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:379 size:1014108528 collection:"collection2" file_count:15362 delete_count:6 deleted_byte_count:2481613 replica_placement:100 version:3 modified_at_second:1615632323 + Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 +total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 64c8094fe..595dbf679 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -3,13 +3,17 @@ package shell import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/cluster" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/util/grace" + "golang.org/x/exp/slices" "io" + "math/rand" "os" "path" "regexp" - "sort" "strings" "github.com/peterh/liner" @@ -21,11 +25,9 @@ var ( ) func RunShell(options ShellOptions) { - - sort.Slice(Commands, func(i, j int) bool { - return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0 + slices.SortFunc(Commands, func(a, b command) bool { + return strings.Compare(a.Name(), b.Name()) < 0 }) - line = liner.NewLiner() defer line.Close() grace.OnInterrupt(func() { @@ -42,13 +44,37 @@ func RunShell(options ShellOptions) { reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) - commandEnv := NewCommandEnv(options) + commandEnv := NewCommandEnv(&options) go commandEnv.MasterClient.KeepConnectedToMaster() commandEnv.MasterClient.WaitUntilConnected() + if commandEnv.option.FilerAddress == "" { + var filers []pb.ServerAddress + commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: *options.FilerGroup, + }) + if err != nil { + return err + } + + for _, clusterNode := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(clusterNode.Address)) + } + return nil + }) + fmt.Printf("master: %s ", *options.Masters) + if len(filers) > 0 { + fmt.Printf("filers: %v", filers) + commandEnv.option.FilerAddress = filers[rand.Intn(len(filers))] + } + fmt.Println() + } + if commandEnv.option.FilerAddress != "" { - commandEnv.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error { + commandEnv.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error { resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return err @@ -84,10 +110,12 @@ https://cloud.seaweedfs.com/ui/%s func processEachCmd(reg *regexp.Regexp, cmd string, commandEnv *CommandEnv) bool { cmds := reg.FindAllString(cmd, -1) + + line.AppendHistory(cmd) + if len(cmds) == 0 { return false } else { - line.AppendHistory(cmd) args := make([]string, len(cmds[1:])) diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot new file mode 100644 index 000000000..4a4ca865d Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot differ diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg new file mode 100644 index 000000000..25691af8f --- /dev/null +++ b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf new file mode 100644 index 000000000..67fa00bf8 Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf differ diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff new file mode 100644 index 000000000..8c54182aa Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff differ diff --git a/weed/static/images/folder.gif b/weed/static/images/folder.gif index 400a1f7ca..6ee082081 100644 Binary files a/weed/static/images/folder.gif and b/weed/static/images/folder.gif differ diff --git a/weed/stats/disk_notsupported.go b/weed/stats/disk_notsupported.go index 3d99e6ce7..2bc24a2f0 100644 --- a/weed/stats/disk_notsupported.go +++ b/weed/stats/disk_notsupported.go @@ -1,3 +1,4 @@ +//go:build openbsd || netbsd || plan9 || solaris // +build openbsd netbsd plan9 solaris package stats diff --git a/weed/stats/disk_supported.go b/weed/stats/disk_supported.go index dff580b5b..e698b0095 100644 --- a/weed/stats/disk_supported.go +++ b/weed/stats/disk_supported.go @@ -1,3 +1,4 @@ +//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris // +build !windows,!openbsd,!netbsd,!plan9,!solaris package stats diff --git a/weed/stats/memory_notsupported.go b/weed/stats/memory_notsupported.go index 2bed95266..2508b561b 100644 --- a/weed/stats/memory_notsupported.go +++ b/weed/stats/memory_notsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package stats diff --git a/weed/stats/memory_supported.go b/weed/stats/memory_supported.go index 91fdd005b..55ad4c6bb 100644 --- a/weed/stats/memory_supported.go +++ b/weed/stats/memory_supported.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package stats diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 48d964418..f0b810608 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -3,8 +3,10 @@ package stats import ( "fmt" "log" + "net" "net/http" "os" + "strconv" "strings" "time" @@ -18,6 +20,46 @@ import ( var ( Gather = prometheus.NewRegistry() + MasterClientConnectCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "SeaweedFS", + Subsystem: "wdclient", + Name: "connect_updates", + Help: "Counter of master client leader updates.", + }, []string{"type"}) + + MasterRaftIsleader = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "master", + Name: "is_leader", + Help: "is leader", + }) + + MasterReceivedHeartbeatCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "SeaweedFS", + Subsystem: "master", + Name: "received_heartbeats", + Help: "Counter of master received heartbeat.", + }, []string{"type"}) + + MasterReplicaPlacementMismatch = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "master", + Name: "replica_placement_mismatch", + Help: "replica placement mismatch", + }, []string{"collection", "id"}) + + MasterLeaderChangeCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "SeaweedFS", + Subsystem: "master", + Name: "leader_changes", + Help: "Counter of master leader changes.", + }, []string{"type"}) + FilerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -35,6 +77,14 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), }, []string{"type"}) + FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "filer", + Name: "last_send_timestamp_of_subscribe", + Help: "The last send timestamp of the filer subscription.", + }, []string{"sourceFiler", "clientName", "path"}) + FilerStoreCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -52,6 +102,14 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), }, []string{"store", "type"}) + FilerSyncOffsetGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "filerSync", + Name: "sync_offset", + Help: "The offset of the filer synchronization service.", + }, []string{"sourceFiler", "targetFiler", "clientName", "path"}) + VolumeServerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -115,7 +173,8 @@ var ( Subsystem: "s3", Name: "request_total", Help: "Counter of s3 requests.", - }, []string{"type", "code"}) + }, []string{"type", "code", "bucket"}) + S3RequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "SeaweedFS", @@ -123,14 +182,22 @@ var ( Name: "request_seconds", Help: "Bucketed histogram of s3 request processing time.", Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), - }, []string{"type"}) + }, []string{"type", "bucket"}) ) func init() { + Gather.MustRegister(MasterClientConnectCounter) + Gather.MustRegister(MasterRaftIsleader) + Gather.MustRegister(MasterReceivedHeartbeatCounter) + Gather.MustRegister(MasterLeaderChangeCounter) + Gather.MustRegister(MasterReplicaPlacementMismatch) + Gather.MustRegister(FilerRequestCounter) Gather.MustRegister(FilerRequestHistogram) Gather.MustRegister(FilerStoreCounter) Gather.MustRegister(FilerStoreHistogram) + Gather.MustRegister(FilerSyncOffsetGauge) + Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge) Gather.MustRegister(collectors.NewGoCollector()) Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) @@ -180,5 +247,5 @@ func SourceName(port uint32) string { if err != nil { return "unknown" } - return fmt.Sprintf("%s:%d", hostname, port) + return net.JoinHostPort(hostname, strconv.Itoa(int(port))) } diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go new file mode 100644 index 000000000..59ae022aa --- /dev/null +++ b/weed/stats/metrics_names.go @@ -0,0 +1,31 @@ +package stats + +// This file contains metric names for all errors +// The naming convention is ErrorSomeThing = "error.some.thing" +const ( + // volume server + ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize" + ErrorSizeMismatch = "errorSizeMismatch" + ErrorCRC = "errorCRC" + ErrorIndexOutOfRange = "errorIndexOutOfRange" + + // master topology + ErrorWriteToLocalDisk = "errorWriteToLocalDisk" + ErrorUnmarshalPairs = "errorUnmarshalPairs" + ErrorWriteToReplicas = "errorWriteToReplicas" + + // master client + FailedToKeepConnected = "failedToKeepConnected" + FailedToSend = "failedToSend" + FailedToReceive = "failedToReceive" + RedirectedToleader = "redirectedToleader" + OnPeerUpdate = "onPeerUpdate" + Failed = "failed" + + // filer handler + ErrorReadNotFound = "read.notfound" + ErrorReadInternal = "read.internalerror" + ErrorWriteEntry = "write.entry.failed" + ErrorReadCache = "read.cache.failed" + ErrorReadStream = "read.stream.failed" +) diff --git a/weed/storage/backend/backend.go b/weed/storage/backend/backend.go index 2dc61d02e..132c67636 100644 --- a/weed/storage/backend/backend.go +++ b/weed/storage/backend/backend.go @@ -25,7 +25,7 @@ type BackendStorageFile interface { type BackendStorage interface { ToProperties() map[string]string NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile - CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) + CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error) DeleteFile(key string) (err error) } diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 3b42429cf..cd5207356 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -70,7 +70,10 @@ func (df *DiskFile) Close() error { } func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) { - return df.fileSize, df.modTime, nil + if df.File == nil { + err = os.ErrInvalid + } + return df.fileSize, df.modTime, err } func (df *DiskFile) Name() string { diff --git a/weed/storage/backend/memory_map/memory_map_other.go b/weed/storage/backend/memory_map/memory_map_other.go index e06a0b59a..63c168f6a 100644 --- a/weed/storage/backend/memory_map/memory_map_other.go +++ b/weed/storage/backend/memory_map/memory_map_other.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package memory_map diff --git a/weed/storage/backend/memory_map/memory_map_windows.go b/weed/storage/backend/memory_map/memory_map_windows.go index 7eb713442..4a1a48f99 100644 --- a/weed/storage/backend/memory_map/memory_map_windows.go +++ b/weed/storage/backend/memory_map/memory_map_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package memory_map diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go index 4706c9334..c68712d58 100644 --- a/weed/storage/backend/s3_backend/s3_backend.go +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -37,6 +37,7 @@ type S3BackendStorage struct { region string bucket string endpoint string + storageClass string conn s3iface.S3API } @@ -48,6 +49,10 @@ func newS3BackendStorage(configuration backend.StringProperties, configPrefix st s.region = configuration.GetString(configPrefix + "region") s.bucket = configuration.GetString(configPrefix + "bucket") s.endpoint = configuration.GetString(configPrefix + "endpoint") + s.storageClass = configuration.GetString(configPrefix + "storage_class") + if s.storageClass == "" { + s.storageClass = "STANDARD_IA" + } s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region, s.endpoint) @@ -62,6 +67,7 @@ func (s *S3BackendStorage) ToProperties() map[string]string { m["region"] = s.region m["bucket"] = s.bucket m["endpoint"] = s.endpoint + m["storage_class"] = s.storageClass return m } @@ -79,13 +85,13 @@ func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb return f } -func (s *S3BackendStorage) CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) { +func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) { randomUuid, _ := uuid.NewRandom() key = randomUuid.String() glog.V(1).Infof("copying dat file of %s to remote s3.%s as %s", f.Name(), s.id, key) - size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, attributes, fn) + size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, s.storageClass, fn) return } diff --git a/weed/storage/backend/s3_backend/s3_download.go b/weed/storage/backend/s3_backend/s3_download.go index dbc28446a..f0cdbefec 100644 --- a/weed/storage/backend/s3_backend/s3_download.go +++ b/weed/storage/backend/s3_backend/s3_download.go @@ -47,7 +47,7 @@ func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string Key: aws.String(sourceKey), }) if err != nil { - return fileSize, fmt.Errorf("failed to download file %s: %v", destFileName, err) + return fileSize, fmt.Errorf("failed to download /buckets/%s%s to %s: %v", sourceBucket, sourceKey, destFileName, err) } glog.V(1).Infof("downloaded file %s\n", destFileName) @@ -58,10 +58,10 @@ func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string // adapted from https://github.com/aws/aws-sdk-go/pull/1868 // and https://petersouter.xyz/s3-download-progress-bar-in-golang/ type s3DownloadProgressedWriter struct { - fp *os.File size int64 written int64 fn func(progressed int64, percentage float32) error + fp *os.File } func (w *s3DownloadProgressedWriter) WriteAt(p []byte, off int64) (int, error) { diff --git a/weed/storage/backend/s3_backend/s3_sessions.go b/weed/storage/backend/s3_backend/s3_sessions.go index e7b3410e6..3170172d4 100644 --- a/weed/storage/backend/s3_backend/s3_sessions.go +++ b/weed/storage/backend/s3_backend/s3_sessions.go @@ -2,6 +2,8 @@ package s3_backend import ( "fmt" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/chrislusf/seaweedfs/weed/util" "sync" "github.com/aws/aws-sdk-go/aws" @@ -34,9 +36,9 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string) } config := &aws.Config{ - Region: aws.String(region), - Endpoint: aws.String(endpoint), - S3ForcePathStyle: aws.Bool(true), + Region: aws.String(region), + Endpoint: aws.String(endpoint), + S3ForcePathStyle: aws.Bool(true), S3DisableContentMD5Validation: aws.Bool(true), } if awsAccessKeyId != "" && awsSecretAccessKey != "" { @@ -47,6 +49,9 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string) if err != nil { return nil, fmt.Errorf("create aws session in region %s: %v", region, err) } + sess.Handlers.Build.PushBack(func(r *request.Request) { + r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER) + }) t := s3.New(sess) diff --git a/weed/storage/backend/s3_backend/s3_upload.go b/weed/storage/backend/s3_backend/s3_upload.go index 500a85590..2910b8339 100644 --- a/weed/storage/backend/s3_backend/s3_upload.go +++ b/weed/storage/backend/s3_backend/s3_upload.go @@ -2,19 +2,16 @@ package s3_backend import ( "fmt" - "os" - "sync/atomic" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "os" + "sync" "github.com/chrislusf/seaweedfs/weed/glog" ) -func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, - attributes map[string]string, - fn func(progressed int64, percentage float32) error) (fileSize int64, err error) { +func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, storageClass string, fn func(progressed int64, percentage float32) error) (fileSize int64, err error) { //open the file f, err := os.Open(filename) @@ -42,31 +39,19 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey }) fileReader := &s3UploadProgressedReader{ - fp: f, - size: fileSize, - read: -fileSize, - fn: fn, - } - - // process tagging - tags := "" - for k, v := range attributes { - if len(tags) > 0 { - tags = tags + "&" - } - tags = tags + k + "=" + v + fp: f, + size: fileSize, + signMap: map[int64]struct{}{}, + fn: fn, } // Upload the file to S3. var result *s3manager.UploadOutput result, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(destBucket), - Key: aws.String(destKey), - Body: fileReader, - ACL: aws.String("private"), - ServerSideEncryption: aws.String("AES256"), - StorageClass: aws.String("STANDARD_IA"), - Tagging: aws.String(tags), + Bucket: aws.String(destBucket), + Key: aws.String(destKey), + Body: fileReader, + StorageClass: aws.String(storageClass), }) //in case it fails to upload @@ -79,11 +64,14 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey } // adapted from https://github.com/aws/aws-sdk-go/pull/1868 +// https://github.com/aws/aws-sdk-go/blob/main/example/service/s3/putObjectWithProcess/putObjWithProcess.go type s3UploadProgressedReader struct { - fp *os.File - size int64 - read int64 - fn func(progressed int64, percentage float32) error + fp *os.File + size int64 + read int64 + signMap map[int64]struct{} + mux sync.Mutex + fn func(progressed int64, percentage float32) error } func (r *s3UploadProgressedReader) Read(p []byte) (int, error) { @@ -96,8 +84,14 @@ func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) { return n, err } - // Got the length have read( or means has uploaded), and you can construct your message - atomic.AddInt64(&r.read, int64(n)) + r.mux.Lock() + // Ignore the first signature call + if _, ok := r.signMap[off]; ok { + r.read += int64(n) + } else { + r.signMap[off] = struct{}{} + } + r.mux.Unlock() if r.fn != nil { read := r.read diff --git a/weed/storage/backend/volume_create.go b/weed/storage/backend/volume_create.go index d4bd8e40f..93719198b 100644 --- a/weed/storage/backend/volume_create.go +++ b/weed/storage/backend/volume_create.go @@ -1,3 +1,4 @@ +//go:build !linux && !windows // +build !linux,!windows package backend diff --git a/weed/storage/backend/volume_create_linux.go b/weed/storage/backend/volume_create_linux.go index 260c2c2a3..9b03dcda2 100644 --- a/weed/storage/backend/volume_create_linux.go +++ b/weed/storage/backend/volume_create_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package backend diff --git a/weed/storage/backend/volume_create_windows.go b/weed/storage/backend/volume_create_windows.go index 7d40ec0d7..0bbb4959b 100644 --- a/weed/storage/backend/volume_create_windows.go +++ b/weed/storage/backend/volume_create_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package backend diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 33dd272ce..289fd3b47 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -2,8 +2,6 @@ package storage import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "io/ioutil" "os" "path/filepath" "strings" @@ -14,11 +12,14 @@ import ( "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/google/uuid" ) type DiskLocation struct { Directory string + DirectoryUuid string IdxDirectory string DiskType types.DiskType MaxVolumeCount int @@ -34,6 +35,29 @@ type DiskLocation struct { isDiskSpaceLow bool } +func GenerateDirUuid(dir string) (dirUuidString string, err error) { + glog.V(1).Infof("Getting uuid of volume directory:%s", dir) + dirUuidString = "" + fileName := dir + "/vol_dir.uuid" + if !util.FileExists(fileName) { + dirUuid, _ := uuid.NewRandom() + dirUuidString = dirUuid.String() + writeErr := util.WriteFile(fileName, []byte(dirUuidString), 0644) + if writeErr != nil { + glog.Warningf("failed to write uuid to %s : %v", fileName, writeErr) + return "", fmt.Errorf("failed to write uuid to %s : %v", fileName, writeErr) + } + } else { + uuidData, readErr := os.ReadFile(fileName) + if readErr != nil { + glog.Warningf("failed to read uuid from %s : %v", fileName, readErr) + return "", fmt.Errorf("failed to read uuid from %s : %v", fileName, readErr) + } + dirUuidString = string(uuidData) + } + return dirUuidString, nil +} + func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation { dir = util.ResolvePath(dir) if idxDir == "" { @@ -41,8 +65,10 @@ func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSp } else { idxDir = util.ResolvePath(idxDir) } + dirUuid, _ := GenerateDirUuid(dir) location := &DiskLocation{ Directory: dir, + DirectoryUuid: dirUuid, IdxDirectory: idxDir, DiskType: diskType, MaxVolumeCount: maxVolumeCount, @@ -85,9 +111,9 @@ func getValidVolumeName(basename string) string { return "" } -func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapKind) bool { - basename := fileInfo.Name() - if fileInfo.IsDir() { +func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind NeedleMapKind, skipIfEcVolumesExists bool) bool { + basename := dirEntry.Name() + if dirEntry.IsDir() { return false } volumeName := getValidVolumeName(basename) @@ -95,10 +121,17 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne return false } + // skip if ec volumes exists + if skipIfEcVolumesExists { + if util.FileExists(l.Directory + "/" + volumeName + ".ecx") { + return false + } + } + // check for incomplete volume noteFile := l.Directory + "/" + volumeName + ".note" if util.FileExists(noteFile) { - note, _ := ioutil.ReadFile(noteFile) + note, _ := os.ReadFile(noteFile) glog.Warningf("volume %s was not completed: %s", volumeName, string(note)) removeVolumeFiles(l.Directory + "/" + volumeName) removeVolumeFiles(l.IdxDirectory + "/" + volumeName) @@ -138,18 +171,18 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) { - task_queue := make(chan os.FileInfo, 10*concurrency) + task_queue := make(chan os.DirEntry, 10*concurrency) go func() { foundVolumeNames := make(map[string]bool) - if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil { - for _, fi := range fileInfos { - volumeName := getValidVolumeName(fi.Name()) + if dirEntries, err := os.ReadDir(l.Directory); err == nil { + for _, entry := range dirEntries { + volumeName := getValidVolumeName(entry.Name()) if volumeName == "" { continue } if _, found := foundVolumeNames[volumeName]; !found { foundVolumeNames[volumeName] = true - task_queue <- fi + task_queue <- entry } } } @@ -162,7 +195,7 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, con go func() { defer wg.Done() for fi := range task_queue { - _ = l.loadExistingVolume(fi, needleMapKind) + _ = l.loadExistingVolume(fi, needleMapKind, true) } }() } @@ -242,18 +275,20 @@ func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e erro func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool { if fileInfo, found := l.LocateVolume(vid); found { - return l.loadExistingVolume(fileInfo, needleMapKind) + return l.loadExistingVolume(fileInfo, needleMapKind, false) } return false } +var ErrVolumeNotFound = fmt.Errorf("volume not found") + func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error { l.volumesLock.Lock() defer l.volumesLock.Unlock() _, ok := l.volumes[vid] if !ok { - return fmt.Errorf("Volume not found, VolumeId: %d", vid) + return ErrVolumeNotFound } _, err := l.deleteVolumeById(vid) return err @@ -265,7 +300,7 @@ func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error { v, ok := l.volumes[vid] if !ok { - return fmt.Errorf("Volume not loaded, VolumeId: %d", vid) + return ErrVolumeNotFound } v.Close() delete(l.volumes, vid) @@ -275,7 +310,7 @@ func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error { func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume { deltaVols := make(map[needle.VolumeId]*Volume, 0) for k, v := range l.volumes { - if v.Collection == collectionName && !v.isCompacting { + if v.Collection == collectionName && !v.isCompacting && !v.isCommitCompacting { deltaVols[k] = v } } @@ -309,6 +344,16 @@ func (l *DiskLocation) VolumesLen() int { return len(l.volumes) } +func (l *DiskLocation) SetStopping() { + l.volumesLock.Lock() + for _, v := range l.volumes { + v.SetStopping() + } + l.volumesLock.Unlock() + + return +} + func (l *DiskLocation) Close() { l.volumesLock.Lock() for _, v := range l.volumes { @@ -325,12 +370,15 @@ func (l *DiskLocation) Close() { return } -func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) { - if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil { - for _, fileInfo := range fileInfos { - volId, _, err := volumeIdFromFileName(fileInfo.Name()) +func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) { + // println("LocateVolume", vid, "on", l.Directory) + if dirEntries, err := os.ReadDir(l.Directory); err == nil { + for _, entry := range dirEntries { + // println("checking", entry.Name(), "...") + volId, _, err := volumeIdFromFileName(entry.Name()) + // println("volId", volId, "err", err) if vid == volId && err == nil { - return fileInfo, true + return entry, true } } } diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go index 91c7d86a6..5fa5316fd 100644 --- a/weed/storage/disk_location_ec.go +++ b/weed/storage/disk_location_ec.go @@ -2,11 +2,10 @@ package storage import ( "fmt" - "io/ioutil" + "golang.org/x/exp/slices" "os" "path" "regexp" - "sort" "strconv" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" @@ -118,25 +117,23 @@ func (l *DiskLocation) loadEcShards(shards []string, collection string, vid need func (l *DiskLocation) loadAllEcShards() (err error) { - fileInfos, err := ioutil.ReadDir(l.Directory) + dirEntries, err := os.ReadDir(l.Directory) if err != nil { return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err) } if l.IdxDirectory != l.Directory { - indexFileInfos, err := ioutil.ReadDir(l.IdxDirectory) + indexDirEntries, err := os.ReadDir(l.IdxDirectory) if err != nil { return fmt.Errorf("load all ec shards in dir %s: %v", l.IdxDirectory, err) } - fileInfos = append(fileInfos, indexFileInfos...) + dirEntries = append(dirEntries, indexDirEntries...) } - - sort.Slice(fileInfos, func(i, j int) bool { - return fileInfos[i].Name() < fileInfos[j].Name() + slices.SortFunc(dirEntries, func(a, b os.DirEntry) bool { + return a.Name() < b.Name() }) - var sameVolumeShards []string var prevVolumeId needle.VolumeId - for _, fileInfo := range fileInfos { + for _, fileInfo := range dirEntries { if fileInfo.IsDir() { continue } diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go index 34b639407..157149865 100644 --- a/weed/storage/erasure_coding/ec_encoder.go +++ b/weed/storage/erasure_coding/ec_encoder.go @@ -220,7 +220,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi processedSize += largeBlockSize * DataShardsCount } for remainingSize > 0 { - encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs) + err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs) if err != nil { return fmt.Errorf("failed to encode small chunk data: %v", err) } diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index a6e880350..4dd07ec40 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -3,10 +3,11 @@ package erasure_coding import ( "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/volume_info" + "golang.org/x/exp/slices" "math" "os" - "sort" "sync" "time" @@ -30,7 +31,7 @@ type EcVolume struct { ecxFileSize int64 ecxCreatedAt time.Time Shards []*EcVolumeShard - ShardLocations map[ShardId][]string + ShardLocations map[ShardId][]pb.ServerAddress ShardLocationsRefreshTime time.Time ShardLocationsLock sync.RWMutex Version needle.Version @@ -69,7 +70,7 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection volume_info.SaveVolumeInfo(dataBaseFileName+".vif", &volume_server_pb.VolumeInfo{Version: uint32(ev.Version)}) } - ev.ShardLocations = make(map[ShardId][]string) + ev.ShardLocations = make(map[ShardId][]pb.ServerAddress) return } @@ -81,9 +82,8 @@ func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool { } } ev.Shards = append(ev.Shards, ecVolumeShard) - sort.Slice(ev.Shards, func(i, j int) bool { - return ev.Shards[i].VolumeId < ev.Shards[j].VolumeId || - ev.Shards[i].VolumeId == ev.Shards[j].VolumeId && ev.Shards[i].ShardId < ev.Shards[j].ShardId + slices.SortFunc(ev.Shards, func(a, b *EcVolumeShard) bool { + return a.VolumeId < b.VolumeId || a.VolumeId == b.VolumeId && a.ShardId < b.ShardId }) return true } diff --git a/weed/storage/needle/43.dat b/weed/storage/needle/43.dat new file mode 100644 index 000000000..7ef8c28ca Binary files /dev/null and b/weed/storage/needle/43.dat differ diff --git a/weed/storage/needle/crc.go b/weed/storage/needle/crc.go index b7d3edd83..65a45293e 100644 --- a/weed/storage/needle/crc.go +++ b/weed/storage/needle/crc.go @@ -21,6 +21,7 @@ func (c CRC) Update(b []byte) CRC { return CRC(crc32.Update(uint32(c), table, b)) } +// Value Deprecated. Just use the raw uint32 value to compare. func (c CRC) Value() uint32 { return uint32(c>>15|c<<17) + 0xa282ead8 } @@ -51,4 +52,4 @@ func (c *CRCwriter) Write(p []byte) (n int, err error) { return } -func (c *CRCwriter) Sum() uint32 { return c.crc.Value() } // final hash +func (c *CRCwriter) Sum() uint32 { return uint32(c.crc) } // final hash diff --git a/weed/storage/needle/needle.go b/weed/storage/needle/needle.go index 845ffdb24..7443796a7 100644 --- a/weed/storage/needle/needle.go +++ b/weed/storage/needle/needle.go @@ -31,9 +31,9 @@ type Needle struct { Data []byte `comment:"The actual file data"` Flags byte `comment:"boolean flags"` //version2 NameSize uint8 //version2 - Name []byte `comment:"maximum 256 characters"` //version2 + Name []byte `comment:"maximum 255 characters"` //version2 MimeSize uint8 //version2 - Mime []byte `comment:"maximum 256 characters"` //version2 + Mime []byte `comment:"maximum 255 characters"` //version2 PairsSize uint16 //version2 Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"` LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 0888c6b7a..a1f517b45 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "mime" "net/http" "path" @@ -75,16 +74,14 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p if mimeType == "application/octet-stream" { mimeType = "" } - if false { - if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); mimeType == "" && !iAmSure || shouldBeCompressed && iAmSure { - // println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType) - if compressedData, err := util.GzipData(pu.Data); err == nil { - if len(compressedData)*10 < len(pu.Data)*9 { - pu.Data = compressedData - pu.IsGzipped = true - } - // println("gzipped data size", len(compressedData)) + if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); shouldBeCompressed && iAmSure { + // println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType) + if compressedData, err := util.GzipData(pu.Data); err == nil { + if len(compressedData)*10 < len(pu.Data)*9 { + pu.Data = compressedData + pu.IsGzipped = true } + // println("gzipped data size", len(compressedData)) } } } @@ -110,7 +107,7 @@ func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error { pu.FileName = "" dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1)) if err == io.EOF || dataSize == sizeLimit+1 { - io.Copy(ioutil.Discard, r.Body) + io.Copy(io.Discard, r.Body) } pu.Data = pu.bytesBuffer.Bytes() r.Body.Close() @@ -120,7 +117,7 @@ func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error { func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { defer func() { if e != nil && r.Body != nil { - io.Copy(ioutil.Discard, r.Body) + io.Copy(io.Discard, r.Body) r.Body.Close() } }() diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read.go similarity index 59% rename from weed/storage/needle/needle_read_write.go rename to weed/storage/needle/needle_read.go index 3b9eb7237..e94c34ff0 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read.go @@ -1,16 +1,14 @@ package needle import ( - "bytes" "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/backend" . "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" "io" - "math" - "sync" ) const ( @@ -31,168 +29,6 @@ func (n *Needle) DiskSize(version Version) int64 { return GetActualSize(n.Size, version) } -var bufPool = sync.Pool{ - New: func() interface{} { - return new(bytes.Buffer) - }, -} - -func (n *Needle) prepareWriteBuffer(version Version, writeBytes *bytes.Buffer) (Size, int64, error) { - writeBytes.Reset() - switch version { - case Version1: - header := make([]byte, NeedleHeaderSize) - CookieToBytes(header[0:CookieSize], n.Cookie) - NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) - n.Size = Size(len(n.Data)) - SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) - size := n.Size - actualSize := NeedleHeaderSize + int64(n.Size) - writeBytes.Write(header) - writeBytes.Write(n.Data) - padding := PaddingLength(n.Size, version) - util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) - writeBytes.Write(header[0 : NeedleChecksumSize+padding]) - return size, actualSize, nil - case Version2, Version3: - header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation - CookieToBytes(header[0:CookieSize], n.Cookie) - NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) - if len(n.Name) >= math.MaxUint8 { - n.NameSize = math.MaxUint8 - } else { - n.NameSize = uint8(len(n.Name)) - } - n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime)) - if n.DataSize > 0 { - n.Size = 4 + Size(n.DataSize) + 1 - if n.HasName() { - n.Size = n.Size + 1 + Size(n.NameSize) - } - if n.HasMime() { - n.Size = n.Size + 1 + Size(n.MimeSize) - } - if n.HasLastModifiedDate() { - n.Size = n.Size + LastModifiedBytesLength - } - if n.HasTtl() { - n.Size = n.Size + TtlBytesLength - } - if n.HasPairs() { - n.Size += 2 + Size(n.PairsSize) - } - } else { - n.Size = 0 - } - SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) - writeBytes.Write(header[0:NeedleHeaderSize]) - if n.DataSize > 0 { - util.Uint32toBytes(header[0:4], n.DataSize) - writeBytes.Write(header[0:4]) - writeBytes.Write(n.Data) - util.Uint8toBytes(header[0:1], n.Flags) - writeBytes.Write(header[0:1]) - if n.HasName() { - util.Uint8toBytes(header[0:1], n.NameSize) - writeBytes.Write(header[0:1]) - writeBytes.Write(n.Name[:n.NameSize]) - } - if n.HasMime() { - util.Uint8toBytes(header[0:1], n.MimeSize) - writeBytes.Write(header[0:1]) - writeBytes.Write(n.Mime) - } - if n.HasLastModifiedDate() { - util.Uint64toBytes(header[0:8], n.LastModified) - writeBytes.Write(header[8-LastModifiedBytesLength : 8]) - } - if n.HasTtl() && n.Ttl != nil { - n.Ttl.ToBytes(header[0:TtlBytesLength]) - writeBytes.Write(header[0:TtlBytesLength]) - } - if n.HasPairs() { - util.Uint16toBytes(header[0:2], n.PairsSize) - writeBytes.Write(header[0:2]) - writeBytes.Write(n.Pairs) - } - } - padding := PaddingLength(n.Size, version) - util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) - if version == Version2 { - writeBytes.Write(header[0 : NeedleChecksumSize+padding]) - } else { - // version3 - util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs) - writeBytes.Write(header[0 : NeedleChecksumSize+TimestampSize+padding]) - } - - return Size(n.DataSize), GetActualSize(n.Size, version), nil - } - - return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version) -} - -func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) { - - if end, _, e := w.GetStat(); e == nil { - defer func(w backend.BackendStorageFile, off int64) { - if err != nil { - if te := w.Truncate(end); te != nil { - glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te) - } - } - }(w, end) - offset = uint64(end) - } else { - err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) - return - } - if offset >= MaxPossibleVolumeSize && n.Size.IsValid() { - err = fmt.Errorf("Volume Size %d Exeededs %d", offset, MaxPossibleVolumeSize) - return - } - - bytesBuffer := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(bytesBuffer) - - size, actualSize, err = n.prepareWriteBuffer(version, bytesBuffer) - - if err == nil { - _, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset)) - } - - return offset, size, actualSize, err -} - -func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) { - - if end, _, e := w.GetStat(); e == nil { - defer func(w backend.BackendStorageFile, off int64) { - if err != nil { - if te := w.Truncate(end); te != nil { - glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te) - } - } - }(w, end) - offset = uint64(end) - } else { - err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) - return - } - - if version == Version3 { - tsOffset := NeedleHeaderSize + size + NeedleChecksumSize - util.Uint64toBytes(dataSlice[tsOffset:tsOffset+TimestampSize], appendAtNs) - } - - if err == nil { - _, err = w.WriteAt(dataSlice, int64(offset)) - } - - return - -} - func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) { dataSize := GetActualSize(size, version) @@ -205,7 +41,7 @@ func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, versi } if err != nil { fileSize, _, _ := r.GetStat() - println("n", n, "dataSize", dataSize, "offset", offset, "fileSize", fileSize) + glog.Errorf("%s read %d dataSize %d offset %d fileSize %d: %v", r.Name(), n, dataSize, offset, fileSize, err) } return dataSlice, err @@ -217,9 +53,11 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio if n.Size != size { // cookie is not always passed in for this API. Use size to do preliminary checking. if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc() glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) return ErrorSizeMismatch } + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc() return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) } switch version { @@ -234,7 +72,9 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio if size > 0 { checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) - if checksum != newChecksum.Value() { + if checksum != newChecksum.Value() && checksum != uint32(newChecksum) { + // the crc.Value() function is to be deprecated. this double checking is for backward compatible. + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorCRC).Inc() return errors.New("CRC error! Data On Disk Corrupted") } n.Checksum = newChecksum @@ -267,10 +107,18 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.DataSize = util.BytesToUint32(bytes[index : index+4]) index = index + 4 if int(n.DataSize)+index > lenBytes { + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 1) } n.Data = bytes[index : index+int(n.DataSize)] index = index + int(n.DataSize) + } + _, err = n.readNeedleDataVersion2NonData(bytes[index:]) + return +} +func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err error) { + lenBytes := len(bytes) + if index < lenBytes { n.Flags = bytes[index] index = index + 1 } @@ -278,7 +126,8 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.NameSize = uint8(bytes[index]) index = index + 1 if int(n.NameSize)+index > lenBytes { - return fmt.Errorf("index out of range %d", 2) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 2) } n.Name = bytes[index : index+int(n.NameSize)] index = index + int(n.NameSize) @@ -287,39 +136,44 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.MimeSize = uint8(bytes[index]) index = index + 1 if int(n.MimeSize)+index > lenBytes { - return fmt.Errorf("index out of range %d", 3) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 3) } n.Mime = bytes[index : index+int(n.MimeSize)] index = index + int(n.MimeSize) } if index < lenBytes && n.HasLastModifiedDate() { if LastModifiedBytesLength+index > lenBytes { - return fmt.Errorf("index out of range %d", 4) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 4) } n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) index = index + LastModifiedBytesLength } if index < lenBytes && n.HasTtl() { if TtlBytesLength+index > lenBytes { - return fmt.Errorf("index out of range %d", 5) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 5) } n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength]) index = index + TtlBytesLength } if index < lenBytes && n.HasPairs() { if 2+index > lenBytes { - return fmt.Errorf("index out of range %d", 6) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 6) } n.PairsSize = util.BytesToUint16(bytes[index : index+2]) index += 2 if int(n.PairsSize)+index > lenBytes { - return fmt.Errorf("index out of range %d", 7) + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() + return index, fmt.Errorf("index out of range %d", 7) } end := index + int(n.PairsSize) n.Pairs = bytes[index:end] index = end } - return nil + return index, nil } func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) { diff --git a/weed/storage/needle/needle_read_page.go b/weed/storage/needle/needle_read_page.go new file mode 100644 index 000000000..e909869f3 --- /dev/null +++ b/weed/storage/needle/needle_read_page.go @@ -0,0 +1,111 @@ +package needle + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + . "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util" + "io" +) + +// ReadNeedleDataInto uses a needle without n.Data to read the content into an io.Writer +func (n *Needle) ReadNeedleDataInto(r backend.BackendStorageFile, volumeOffset int64, buf []byte, writer io.Writer, needleOffset int64, size int64) (err error) { + crc := CRC(0) + for x := needleOffset; x < needleOffset+size; x += int64(len(buf)) { + count, err := n.ReadNeedleData(r, volumeOffset, buf, x) + toWrite := min(int64(count), needleOffset+size-x) + if toWrite > 0 { + crc = crc.Update(buf[0:toWrite]) + if _, err = writer.Write(buf[0:toWrite]); err != nil { + return fmt.Errorf("ReadNeedleData write: %v", err) + } + } + if err != nil { + if err == io.EOF { + err = nil + break + } + return fmt.Errorf("ReadNeedleData: %v", err) + } + if count <= 0 { + break + } + } + if needleOffset == 0 && size == int64(n.DataSize) && (n.Checksum != crc && uint32(n.Checksum) != crc.Value()) { + // the crc.Value() function is to be deprecated. this double checking is for backward compatible. + return fmt.Errorf("ReadNeedleData checksum %v expected %v", crc, n.Checksum) + } + return nil +} + +// ReadNeedleData uses a needle without n.Data to read the content +// volumeOffset: the offset within the volume +// needleOffset: the offset within the needle Data +func (n *Needle) ReadNeedleData(r backend.BackendStorageFile, volumeOffset int64, data []byte, needleOffset int64) (count int, err error) { + + sizeToRead := min(int64(len(data)), int64(n.DataSize)-needleOffset) + if sizeToRead <= 0 { + return 0, io.EOF + } + startOffset := volumeOffset + NeedleHeaderSize + DataSizeSize + needleOffset + + count, err = r.ReadAt(data[:sizeToRead], startOffset) + if err != nil { + fileSize, _, _ := r.GetStat() + glog.Errorf("%s read %d %d size %d at offset %d fileSize %d: %v", r.Name(), n.Id, needleOffset, sizeToRead, volumeOffset, fileSize, err) + } + return + +} + +// ReadNeedleMeta fills all metadata except the n.Data +func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error) { + + bytes := make([]byte, NeedleHeaderSize+DataSizeSize) + + count, err := r.ReadAt(bytes, offset) + if count != NeedleHeaderSize+DataSizeSize || err != nil { + return err + } + n.ParseNeedleHeader(bytes) + if n.Size != size { + if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { + return ErrorSizeMismatch + } + } + + n.DataSize = util.BytesToUint32(bytes[NeedleHeaderSize : NeedleHeaderSize+DataSizeSize]) + + startOffset := offset + NeedleHeaderSize + DataSizeSize + int64(n.DataSize) + dataSize := GetActualSize(size, version) + stopOffset := offset + dataSize + metaSize := stopOffset - startOffset + metaSlice := make([]byte, int(metaSize)) + + count, err = r.ReadAt(metaSlice, startOffset) + if err != nil && int64(count) == metaSize { + err = nil + } + if err != nil { + return err + } + + var index int + index, err = n.readNeedleDataVersion2NonData(metaSlice) + + n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize])) + if version == Version3 { + n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize]) + } + + return err + +} + +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} diff --git a/weed/storage/needle/needle_read_test.go b/weed/storage/needle/needle_read_test.go new file mode 100644 index 000000000..001f10771 --- /dev/null +++ b/weed/storage/needle/needle_read_test.go @@ -0,0 +1,82 @@ +package needle + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + "io" + "os" + "testing" + + "github.com/chrislusf/seaweedfs/weed/storage/types" +) + +func TestPageRead(t *testing.T) { + baseFileName := "43" + offset := int64(8) + size := types.Size(1153890) // actual file size 1153862 + + datFile, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0644) + if err != nil { + t.Fatalf("Open Volume Data File [ERROR]: %v", err) + } + datBackend := backend.NewDiskFile(datFile) + defer datBackend.Close() + { + n := new(Needle) + + bytes, err := ReadNeedleBlob(datBackend, offset, size, Version3) + if err != nil { + t.Fatalf("readNeedleBlob: %v", err) + } + if err = n.ReadBytes(bytes, offset, size, Version3); err != nil { + t.Fatalf("readNeedleBlob: %v", err) + } + + fmt.Printf("bytes len %d\n", len(bytes)) + fmt.Printf("name %s size %d\n", n.Name, n.Size) + + fmt.Printf("id %d\n", n.Id) + fmt.Printf("DataSize %d\n", n.DataSize) + fmt.Printf("Flags %v\n", n.Flags) + fmt.Printf("NameSize %d\n", n.NameSize) + fmt.Printf("MimeSize %d\n", n.MimeSize) + fmt.Printf("PairsSize %d\n", n.PairsSize) + fmt.Printf("LastModified %d\n", n.LastModified) + fmt.Printf("AppendAtNs %d\n", n.AppendAtNs) + fmt.Printf("Checksum %d\n", n.Checksum) + } + + { + n, bytes, bodyLength, err := ReadNeedleHeader(datBackend, Version3, offset) + if err != nil { + t.Fatalf("ReadNeedleHeader: %v", err) + } + fmt.Printf("bytes len %d\n", len(bytes)) + fmt.Printf("name %s size %d bodyLength:%d\n", n.Name, n.Size, bodyLength) + } + + { + n := new(Needle) + err := n.ReadNeedleMeta(datBackend, offset, size, Version3) + if err != nil { + t.Fatalf("ReadNeedleHeader: %v", err) + } + fmt.Printf("name %s size %d\n", n.Name, n.Size) + fmt.Printf("id %d\n", n.Id) + fmt.Printf("DataSize %d\n", n.DataSize) + fmt.Printf("Flags %v\n", n.Flags) + fmt.Printf("NameSize %d\n", n.NameSize) + fmt.Printf("MimeSize %d\n", n.MimeSize) + fmt.Printf("PairsSize %d\n", n.PairsSize) + fmt.Printf("LastModified %d\n", n.LastModified) + fmt.Printf("AppendAtNs %d\n", n.AppendAtNs) + fmt.Printf("Checksum %d\n", n.Checksum) + + buf := make([]byte, 1024) + if err = n.ReadNeedleDataInto(datBackend, offset, buf, io.Discard, 0, int64(n.DataSize)); err != nil { + t.Fatalf("ReadNeedleDataInto: %v", err) + } + + } + +} diff --git a/weed/storage/needle/needle_write.go b/weed/storage/needle/needle_write.go new file mode 100644 index 000000000..fd2e33ad9 --- /dev/null +++ b/weed/storage/needle/needle_write.go @@ -0,0 +1,174 @@ +package needle + +import ( + "bytes" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + . "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util" + "math" + "sync" +) + +var bufPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, +} + +func (n *Needle) prepareWriteBuffer(version Version, writeBytes *bytes.Buffer) (Size, int64, error) { + writeBytes.Reset() + switch version { + case Version1: + header := make([]byte, NeedleHeaderSize) + CookieToBytes(header[0:CookieSize], n.Cookie) + NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) + n.Size = Size(len(n.Data)) + SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) + size := n.Size + actualSize := NeedleHeaderSize + int64(n.Size) + writeBytes.Write(header) + writeBytes.Write(n.Data) + padding := PaddingLength(n.Size, version) + util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum)) + writeBytes.Write(header[0 : NeedleChecksumSize+padding]) + return size, actualSize, nil + case Version2, Version3: + header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation + CookieToBytes(header[0:CookieSize], n.Cookie) + NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) + if len(n.Name) >= math.MaxUint8 { + n.NameSize = math.MaxUint8 + } else { + n.NameSize = uint8(len(n.Name)) + } + n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime)) + if n.DataSize > 0 { + n.Size = 4 + Size(n.DataSize) + 1 + if n.HasName() { + n.Size = n.Size + 1 + Size(n.NameSize) + } + if n.HasMime() { + n.Size = n.Size + 1 + Size(n.MimeSize) + } + if n.HasLastModifiedDate() { + n.Size = n.Size + LastModifiedBytesLength + } + if n.HasTtl() { + n.Size = n.Size + TtlBytesLength + } + if n.HasPairs() { + n.Size += 2 + Size(n.PairsSize) + } + } else { + n.Size = 0 + } + SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) + writeBytes.Write(header[0:NeedleHeaderSize]) + if n.DataSize > 0 { + util.Uint32toBytes(header[0:4], n.DataSize) + writeBytes.Write(header[0:4]) + writeBytes.Write(n.Data) + util.Uint8toBytes(header[0:1], n.Flags) + writeBytes.Write(header[0:1]) + if n.HasName() { + util.Uint8toBytes(header[0:1], n.NameSize) + writeBytes.Write(header[0:1]) + writeBytes.Write(n.Name[:n.NameSize]) + } + if n.HasMime() { + util.Uint8toBytes(header[0:1], n.MimeSize) + writeBytes.Write(header[0:1]) + writeBytes.Write(n.Mime) + } + if n.HasLastModifiedDate() { + util.Uint64toBytes(header[0:8], n.LastModified) + writeBytes.Write(header[8-LastModifiedBytesLength : 8]) + } + if n.HasTtl() && n.Ttl != nil { + n.Ttl.ToBytes(header[0:TtlBytesLength]) + writeBytes.Write(header[0:TtlBytesLength]) + } + if n.HasPairs() { + util.Uint16toBytes(header[0:2], n.PairsSize) + writeBytes.Write(header[0:2]) + writeBytes.Write(n.Pairs) + } + } + padding := PaddingLength(n.Size, version) + util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum)) + if version == Version2 { + writeBytes.Write(header[0 : NeedleChecksumSize+padding]) + } else { + // version3 + util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs) + writeBytes.Write(header[0 : NeedleChecksumSize+TimestampSize+padding]) + } + + return Size(n.DataSize), GetActualSize(n.Size, version), nil + } + + return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version) +} + +func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) { + + if end, _, e := w.GetStat(); e == nil { + defer func(w backend.BackendStorageFile, off int64) { + if err != nil { + if te := w.Truncate(end); te != nil { + glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te) + } + } + }(w, end) + offset = uint64(end) + } else { + err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) + return + } + if offset >= MaxPossibleVolumeSize && n.Size.IsValid() { + err = fmt.Errorf("Volume Size %d Exeededs %d", offset, MaxPossibleVolumeSize) + return + } + + bytesBuffer := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(bytesBuffer) + + size, actualSize, err = n.prepareWriteBuffer(version, bytesBuffer) + + if err == nil { + _, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset)) + } + + return offset, size, actualSize, err +} + +func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) { + + if end, _, e := w.GetStat(); e == nil { + defer func(w backend.BackendStorageFile, off int64) { + if err != nil { + if te := w.Truncate(end); te != nil { + glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te) + } + } + }(w, end) + offset = uint64(end) + } else { + err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) + return + } + + if version == Version3 { + tsOffset := NeedleHeaderSize + size + NeedleChecksumSize + util.Uint64toBytes(dataSlice[tsOffset:tsOffset+TimestampSize], appendAtNs) + } + + if err == nil { + _, err = w.WriteAt(dataSlice, int64(offset)) + } + + return + +} diff --git a/weed/storage/needle/needle_read_write_test.go b/weed/storage/needle/needle_write_test.go similarity index 97% rename from weed/storage/needle/needle_read_write_test.go rename to weed/storage/needle/needle_write_test.go index afcea5a05..20dbc2554 100644 --- a/weed/storage/needle/needle_read_write_test.go +++ b/weed/storage/needle/needle_write_test.go @@ -1,7 +1,6 @@ package needle import ( - "io/ioutil" "os" "testing" @@ -31,7 +30,7 @@ func TestAppend(t *testing.T) { Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"` } - tempFile, err := ioutil.TempFile("", ".dat") + tempFile, err := os.CreateTemp("", ".dat") if err != nil { t.Errorf("Fail TempFile. %v", err) return diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go index 3d2047f99..ccce8f108 100644 --- a/weed/storage/needle_map/compact_map.go +++ b/weed/storage/needle_map/compact_map.go @@ -8,7 +8,7 @@ import ( ) const ( - batch = 100000 + batch = 10000 ) type SectionalNeedleId uint32 diff --git a/weed/storage/needle_map/compact_map_cases_test.go b/weed/storage/needle_map/compact_map_cases_test.go index 305925699..83376e3e8 100644 --- a/weed/storage/needle_map/compact_map_cases_test.go +++ b/weed/storage/needle_map/compact_map_cases_test.go @@ -1,3 +1,4 @@ +//go:build 5BytesOffset // +build 5BytesOffset package needle_map diff --git a/weed/storage/needle_map/compact_map_test.go b/weed/storage/needle_map/compact_map_test.go index 199cb26b3..afe12ee72 100644 --- a/weed/storage/needle_map/compact_map_test.go +++ b/weed/storage/needle_map/compact_map_test.go @@ -2,10 +2,25 @@ package needle_map import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/sequence" . "github.com/chrislusf/seaweedfs/weed/storage/types" "testing" ) +func TestSnowflakeSequencer(t *testing.T) { + m := NewCompactMap() + seq, _ := sequence.NewSnowflakeSequencer("for_test", 1) + + for i := 0; i < 200000; i++ { + id := seq.NextFileId(1) + oldOffset, oldSize := m.Set(NeedleId(id), ToOffset(8), 3000073) + if oldSize != 0 { + t.Errorf("id %d oldOffset %v oldSize %d", id, oldOffset, oldSize) + } + } + +} + func TestOverflow2(t *testing.T) { m := NewCompactMap() _, oldSize := m.Set(NeedleId(150088), ToOffset(8), 3000073) diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go index 7cf48dd72..0bdfdcedc 100644 --- a/weed/storage/needle_map_metric.go +++ b/weed/storage/needle_map_metric.go @@ -93,10 +93,10 @@ func (mm *mapMetric) MaybeSetMaxFileKey(key NeedleId) { func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) { mm = &mapMetric{} - var bf *boom.StableBloomFilter + var bf *boom.BloomFilter buf := make([]byte, NeedleIdSize) err = reverseWalkIndexFile(r, func(entryCount int64) { - bf = boom.NewDefaultStableBloomFilter(uint(entryCount), 0.001) + bf = boom.NewBloomFilter(uint(entryCount), 0.001) }, func(key NeedleId, offset Offset, size Size) error { mm.MaybeSetMaxFileKey(key) @@ -105,9 +105,8 @@ func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) { mm.FileByteCounter += uint64(size) } - if !bf.Test(buf) { + if !bf.TestAndAdd(buf) { mm.FileCounter++ - bf.Add(buf) } else { // deleted file mm.DeletionCounter++ diff --git a/weed/storage/needle_map_metric_test.go b/weed/storage/needle_map_metric_test.go index 362659a11..c04fd6c8b 100644 --- a/weed/storage/needle_map_metric_test.go +++ b/weed/storage/needle_map_metric_test.go @@ -1,8 +1,8 @@ package storage import ( - "io/ioutil" "math/rand" + "os" "testing" "github.com/chrislusf/seaweedfs/weed/glog" @@ -11,7 +11,7 @@ import ( func TestFastLoadingNeedleMapMetrics(t *testing.T) { - idxFile, _ := ioutil.TempFile("", "tmp.idx") + idxFile, _ := os.CreateTemp("", "tmp.idx") nm := NewCompactNeedleMap(idxFile) for i := 0; i < 10000; i++ { diff --git a/weed/storage/store.go b/weed/storage/store.go index 044c31050..f833f1b15 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -2,12 +2,15 @@ package storage import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/volume_info" - "github.com/chrislusf/seaweedfs/weed/util" + "io" "path/filepath" "strings" "sync/atomic" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/storage/volume_info" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/glog" @@ -24,18 +27,26 @@ const ( ) type ReadOption struct { - ReadDeleted bool + // request + ReadDeleted bool + AttemptMetaOnly bool + MustMetaOnly bool + // response + IsMetaOnly bool // read status + VolumeRevision uint16 + IsOutOfRange bool // whether read over MaxPossibleVolumeSize } /* * A VolumeServer contains one Store */ type Store struct { - MasterAddress string + MasterAddress pb.ServerAddress grpcDialOption grpc.DialOption volumeSizeLimit uint64 // read from the master Ip string Port int + GrpcPort int PublicUrl string Locations []*DiskLocation dataCenter string // optional informaton, overwriting master setting if exists @@ -50,13 +61,13 @@ type Store struct { } func (s *Store) String() (str string) { - str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.GetVolumeSizeLimit()) + str = fmt.Sprintf("Ip:%s, Port:%d, GrpcPort:%d PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d", s.Ip, s.Port, s.GrpcPort, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.GetVolumeSizeLimit()) return } -func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, +func NewStore(grpcDialOption grpc.DialOption, ip string, port int, grpcPort int, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) { - s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind} + s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, GrpcPort: grpcPort, PublicUrl: publicUrl, NeedleMapKind: needleMapKind} s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirnames); i++ { location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpaces[i], idxFolder, diskTypes[i]) @@ -298,6 +309,11 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { } } + var uuidList []string + for _, loc := range s.Locations { + uuidList = append(uuidList, loc.DirectoryUuid) + } + for col, size := range collectionVolumeSize { stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size)) } @@ -311,6 +327,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { return &master_pb.Heartbeat{ Ip: s.Ip, Port: uint32(s.Port), + GrpcPort: uint32(s.GrpcPort), PublicUrl: s.PublicUrl, MaxVolumeCounts: maxVolumeCounts, MaxFileKey: NeedleIdToUint64(maxFileKey), @@ -318,12 +335,16 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { Rack: s.rack, Volumes: volumeMessages, HasNoVolumes: len(volumeMessages) == 0, + LocationUuids: uuidList, } } func (s *Store) SetStopping() { s.isStopping = true + for _, location := range s.Locations { + location.SetStopping() + } } func (s *Store) Close() { @@ -362,6 +383,12 @@ func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle, readOption } return 0, fmt.Errorf("volume %d not found", i) } +func (s *Store) ReadVolumeNeedleDataInto(i needle.VolumeId, n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) error { + if v := s.findVolume(i); v != nil { + return v.readNeedleDataInto(n, readOption, writer, offset, size) + } + return fmt.Errorf("volume %d not found", i) +} func (s *Store) GetVolume(i needle.VolumeId) *Volume { return s.findVolume(i) } @@ -428,10 +455,13 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error { } for _, location := range s.Locations { - if err := location.UnloadVolume(i); err == nil { + err := location.UnloadVolume(i) + if err == nil { glog.V(0).Infof("UnmountVolume %d", i) s.DeletedVolumesChan <- message return nil + } else if err == ErrVolumeNotFound { + continue } } @@ -452,10 +482,13 @@ func (s *Store) DeleteVolume(i needle.VolumeId) error { DiskType: string(v.location.DiskType), } for _, location := range s.Locations { - if err := location.DeleteVolume(i); err == nil { + err := location.DeleteVolume(i) + if err == nil { glog.V(0).Infof("DeleteVolume %d", i) s.DeletedVolumesChan <- message return nil + } else if err == ErrVolumeNotFound { + continue } else { glog.Errorf("DeleteVolume %d: %v", i, err) } diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 6ba7237e2..0c9de45aa 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -3,9 +3,10 @@ package storage import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "golang.org/x/exp/slices" "io" "os" - "sort" "sync" "time" @@ -238,7 +239,7 @@ func (s *Store) cachedLookupEcShardLocations(ecVolume *erasure_coding.EcVolume) glog.V(3).Infof("lookup and cache ec volume %d locations", ecVolume.VolumeId) - err = operation.WithMasterServerClient(s.MasterAddress, s.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + err = operation.WithMasterServerClient(false, s.MasterAddress, s.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { req := &master_pb.LookupEcVolumeRequest{ VolumeId: uint32(ecVolume.VolumeId), } @@ -255,7 +256,7 @@ func (s *Store) cachedLookupEcShardLocations(ecVolume *erasure_coding.EcVolume) shardId := erasure_coding.ShardId(shardIdLocations.ShardId) delete(ecVolume.ShardLocations, shardId) for _, loc := range shardIdLocations.Locations { - ecVolume.ShardLocations[shardId] = append(ecVolume.ShardLocations[shardId], loc.Url) + ecVolume.ShardLocations[shardId] = append(ecVolume.ShardLocations[shardId], pb.NewServerAddressFromLocation(loc)) } } ecVolume.ShardLocationsRefreshTime = time.Now() @@ -266,7 +267,7 @@ func (s *Store) cachedLookupEcShardLocations(ecVolume *erasure_coding.EcVolume) return } -func (s *Store) readRemoteEcShardInterval(sourceDataNodes []string, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) { +func (s *Store) readRemoteEcShardInterval(sourceDataNodes []pb.ServerAddress, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) { if len(sourceDataNodes) == 0 { return 0, false, fmt.Errorf("failed to find ec shard %d.%d", vid, shardId) @@ -284,9 +285,9 @@ func (s *Store) readRemoteEcShardInterval(sourceDataNodes []string, needleId typ return } -func (s *Store) doReadRemoteEcShardInterval(sourceDataNode string, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) { +func (s *Store) doReadRemoteEcShardInterval(sourceDataNode pb.ServerAddress, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) { - err = operation.WithVolumeServerClient(sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { // copy data slice shardReadClient, err := client.VolumeEcShardRead(context.Background(), &volume_server_pb.VolumeEcShardReadRequest{ @@ -349,7 +350,7 @@ func (s *Store) recoverOneRemoteEcShardInterval(needleId types.NeedleId, ecVolum // read from remote locations wg.Add(1) - go func(shardId erasure_coding.ShardId, locations []string) { + go func(shardId erasure_coding.ShardId, locations []pb.ServerAddress) { defer wg.Done() data := make([]byte, len(buf)) nRead, isDeleted, readErr := s.readRemoteEcShardInterval(locations, needleId, ecVolume.VolumeId, shardId, data, offset) @@ -388,8 +389,8 @@ func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) { } location.ecVolumesLock.RUnlock() } - sort.Slice(ecVolumes, func(i, j int) bool { - return ecVolumes[i].VolumeId > ecVolumes[j].VolumeId + slices.SortFunc(ecVolumes, func(a, b *erasure_coding.EcVolume) bool { + return a.VolumeId > b.VolumeId }) return ecVolumes } diff --git a/weed/storage/store_ec_delete.go b/weed/storage/store_ec_delete.go index 6c10af3c5..398771a83 100644 --- a/weed/storage/store_ec_delete.go +++ b/weed/storage/store_ec_delete.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" @@ -85,9 +86,9 @@ func (s *Store) doDeleteNeedleFromRemoteEcShardServers(shardId erasure_coding.Sh } -func (s *Store) doDeleteNeedleFromRemoteEcShard(sourceDataNode string, vid needle.VolumeId, collection string, version needle.Version, needleId types.NeedleId) error { +func (s *Store) doDeleteNeedleFromRemoteEcShard(sourceDataNode pb.ServerAddress, vid needle.VolumeId, collection string, version needle.Version, needleId types.NeedleId) error { - return operation.WithVolumeServerClient(sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, sourceDataNode, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { // copy data slice _, err := client.VolumeEcBlobDelete(context.Background(), &volume_server_pb.VolumeEcBlobDeleteRequest{ diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go index fe2033070..cbd716b32 100644 --- a/weed/storage/store_vacuum.go +++ b/weed/storage/store_vacuum.go @@ -15,17 +15,20 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) { } return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId) } -func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64) error { +func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error { if v := s.findVolume(vid); v != nil { s := stats.NewDiskStatus(v.dir) if int64(s.Free) < preallocate { return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate) } - return v.Compact2(preallocate, compactionBytePerSecond) + return v.Compact2(preallocate, compactionBytePerSecond, progressFn) } return fmt.Errorf("volume id %d is not found during compact", vid) } func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, error) { + if s.isStopping { + return false, fmt.Errorf("volume id %d skips compact because volume is stopping", vid) + } if v := s.findVolume(vid); v != nil { return v.IsReadOnly(), v.CommitCompact() } diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go index 137b97d7f..98287fe6a 100644 --- a/weed/storage/types/needle_types.go +++ b/weed/storage/types/needle_types.go @@ -33,6 +33,7 @@ type Cookie uint32 const ( SizeSize = 4 // uint32 size NeedleHeaderSize = CookieSize + NeedleIdSize + SizeSize + DataSizeSize = 4 NeedleMapEntrySize = NeedleIdSize + OffsetSize + SizeSize TimestampSize = 8 // int64 size NeedlePaddingSize = 8 diff --git a/weed/storage/types/offset_4bytes.go b/weed/storage/types/offset_4bytes.go index 5348d5b36..5835b24d7 100644 --- a/weed/storage/types/offset_4bytes.go +++ b/weed/storage/types/offset_4bytes.go @@ -1,3 +1,4 @@ +//go:build !5BytesOffset // +build !5BytesOffset package types diff --git a/weed/storage/types/offset_5bytes.go b/weed/storage/types/offset_5bytes.go index b6181fc11..4b16bd822 100644 --- a/weed/storage/types/offset_5bytes.go +++ b/weed/storage/types/offset_5bytes.go @@ -1,3 +1,4 @@ +//go:build 5BytesOffset // +build 5BytesOffset package types diff --git a/weed/storage/volume.go b/weed/storage/volume.go index c6bf3e329..3539efa85 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -42,7 +42,8 @@ type Volume struct { lastCompactIndexOffset uint64 lastCompactRevision uint16 - isCompacting bool + isCompacting bool + isCommitCompacting bool volumeInfo *volume_server_pb.VolumeInfo location *DiskLocation @@ -175,10 +176,46 @@ func (v *Volume) DiskType() types.DiskType { return v.location.DiskType } +func (v *Volume) SetStopping() { + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() + if v.nm != nil { + if err := v.nm.Sync(); err != nil { + glog.Warningf("Volume SetStopping fail to sync volume idx %d", v.Id) + } + } + if v.DataBackend != nil { + if err := v.DataBackend.Sync(); err != nil { + glog.Warningf("Volume SetStopping fail to sync volume %d", v.Id) + } + } +} + +func (v *Volume) SyncToDisk() { + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() + if v.nm != nil { + if err := v.nm.Sync(); err != nil { + glog.Warningf("Volume Close fail to sync volume idx %d", v.Id) + } + } + if v.DataBackend != nil { + if err := v.DataBackend.Sync(); err != nil { + glog.Warningf("Volume Close fail to sync volume %d", v.Id) + } + } +} + // Close cleanly shuts down this volume func (v *Volume) Close() { v.dataFileAccessLock.Lock() defer v.dataFileAccessLock.Unlock() + + for v.isCommitCompacting { + time.Sleep(521 * time.Millisecond) + glog.Warningf("Volume Close wait for compaction %d", v.Id) + } + if v.nm != nil { if err := v.nm.Sync(); err != nil { glog.Warningf("Volume Close fail to sync volume idx %d", v.Id) @@ -245,7 +282,7 @@ func (v *Volume) collectStatus() (maxFileKey types.NeedleId, datFileSize int64, defer v.dataFileAccessLock.RUnlock() glog.V(3).Infof("collectStatus volume %d", v.Id) - if v.nm == nil { + if v.nm == nil || v.DataBackend == nil { return } diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go index 82ea12a89..67406aff6 100644 --- a/weed/storage/volume_backup.go +++ b/weed/storage/volume_backup.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" "io" "os" @@ -62,7 +63,7 @@ update needle map when receiving new .dat bytes. But seems not necessary now.) */ -func (v *Volume) IncrementalBackup(volumeServer string, grpcDialOption grpc.DialOption) error { +func (v *Volume) IncrementalBackup(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption) error { startFromOffset, _, _ := v.FileStat() appendAtNs, err := v.findLastAppendAtNs() @@ -72,7 +73,7 @@ func (v *Volume) IncrementalBackup(volumeServer string, grpcDialOption grpc.Dial writeOffset := int64(startFromOffset) - err = operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + err = operation.WithVolumeServerClient(false, volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { stream, err := client.VolumeIncrementalCopy(context.Background(), &volume_server_pb.VolumeIncrementalCopyRequest{ VolumeId: uint32(v.Id), @@ -190,12 +191,42 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast // read the appendAtNs for entry m offset, err = v.readOffsetFromIndex(m) if err != nil { + err = fmt.Errorf("read entry %d: %v", m, err) return } + if offset.IsZero() { + leftIndex, _, leftNs, leftErr := v.readLeftNs(m) + if leftErr != nil { + err = leftErr + return + } + rightIndex, rightOffset, rightNs, rightErr := v.readRightNs(m) + if rightErr != nil { + err = rightErr + return + } + if rightNs <= sinceNs { + l = rightIndex + if l == entryCount { + return Offset{}, true, nil + } else { + continue + } + } + if sinceNs < leftNs { + h = leftIndex + 1 + continue + } + return rightOffset, false, nil + + } + if offset.IsZero() { + return Offset{}, true, nil + } mNs, nsReadErr := v.readAppendAtNs(offset) if nsReadErr != nil { - err = nsReadErr + err = fmt.Errorf("read entry %d offset %d: %v", m, offset, nsReadErr) return } @@ -218,6 +249,38 @@ func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast } +func (v *Volume) readRightNs(m int64) (index int64, offset Offset, ts uint64, err error) { + index = m + for offset.IsZero() { + index++ + offset, err = v.readOffsetFromIndex(index) + if err != nil { + err = fmt.Errorf("read entry %d: %v", index, err) + return + } + } + if !offset.IsZero() { + ts, err = v.readAppendAtNs(offset) + } + return +} + +func (v *Volume) readLeftNs(m int64) (index int64, offset Offset, ts uint64, err error) { + index = m + for offset.IsZero() { + index-- + offset, err = v.readOffsetFromIndex(index) + if err != nil { + err = fmt.Errorf("read entry %d: %v", index, err) + return + } + } + if !offset.IsZero() { + ts, err = v.readAppendAtNs(offset) + } + return +} + // bytes is of size NeedleMapEntrySize func (v *Volume) readOffsetFromIndex(m int64) (Offset, error) { v.dataFileAccessLock.RLock() diff --git a/weed/storage/volume_info/volume_info.go b/weed/storage/volume_info/volume_info.go index b7ef75171..aa815970c 100644 --- a/weed/storage/volume_info/volume_info.go +++ b/weed/storage/volume_info/volume_info.go @@ -3,15 +3,14 @@ package volume_info import ( "bytes" "fmt" - "io/ioutil" - - _ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend" - "github.com/chrislusf/seaweedfs/weed/util" + "os" "github.com/golang/protobuf/jsonpb" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + _ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend" + "github.com/chrislusf/seaweedfs/weed/util" ) // MaybeLoadVolumeInfo load the file data as *volume_server_pb.VolumeInfo, the returned volumeInfo will not be nil @@ -36,7 +35,7 @@ func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeIn hasVolumeInfoFile = true glog.V(1).Infof("maybeLoadVolumeInfo reads %s", fileName) - tierData, readErr := ioutil.ReadFile(fileName) + tierData, readErr := os.ReadFile(fileName) if readErr != nil { glog.Warningf("fail to read %s : %v", fileName, readErr) err = fmt.Errorf("fail to read %s : %v", fileName, readErr) @@ -76,7 +75,7 @@ func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) er return fmt.Errorf("marshal to %s: %v", fileName, marshalErr) } - writeErr := ioutil.WriteFile(fileName, []byte(text), 0755) + writeErr := util.WriteFile(fileName, []byte(text), 0755) if writeErr != nil { return fmt.Errorf("fail to write %s : %v", fileName, writeErr) } diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index 0cf603ad8..2dc75d5ee 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -83,6 +83,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind if alreadyHasSuperBlock { err = v.readSuperBlock() + if err == nil { + v.volumeInfo.Version = uint32(v.SuperBlock.Version) + } glog.V(0).Infof("readSuperBlock volume %d version %v", v.Id, v.SuperBlock.Version) if v.HasRemoteFile() { // maybe temporary network problem diff --git a/weed/storage/volume_read.go b/weed/storage/volume_read.go index 9751b56ae..6b17eea0a 100644 --- a/weed/storage/volume_read.go +++ b/weed/storage/volume_read.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/util/mem" "io" "time" @@ -12,8 +13,10 @@ import ( . "github.com/chrislusf/seaweedfs/weed/storage/types" ) +const PagedReadLimit = 1024 * 1024 + // read fills in Needle content by looking up n.Id from NeedleMapper -func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSizeFn func(size Size)) (int, error) { +func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSizeFn func(size Size)) (count int, err error) { v.dataFileAccessLock.RLock() defer v.dataFileAccessLock.RUnlock() @@ -36,31 +39,84 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize if onReadSizeFn != nil { onReadSizeFn(readSize) } - err := n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) - if err == needle.ErrorSizeMismatch && OffsetSize == 4 { - err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) + if readOption != nil && readOption.AttemptMetaOnly && readSize > PagedReadLimit { + readOption.VolumeRevision = v.SuperBlock.CompactionRevision + err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) + if err == needle.ErrorSizeMismatch && OffsetSize == 4 { + readOption.IsOutOfRange = true + err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) + } + if err != nil { + return 0, err + } + if !n.IsCompressed() && !n.IsChunkedManifest() { + readOption.IsMetaOnly = true + } } - v.checkReadWriteError(err) - if err != nil { - return 0, err + if readOption == nil || !readOption.IsMetaOnly { + err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) + if err == needle.ErrorSizeMismatch && OffsetSize == 4 { + err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) + } + v.checkReadWriteError(err) + if err != nil { + return 0, err + } } - bytesRead := len(n.Data) + count = int(n.DataSize) if !n.HasTtl() { - return bytesRead, nil + return } ttlMinutes := n.Ttl.Minutes() if ttlMinutes == 0 { - return bytesRead, nil + return } if !n.HasLastModifiedDate() { - return bytesRead, nil + return } if time.Now().Before(time.Unix(0, int64(n.AppendAtNs)).Add(time.Duration(ttlMinutes) * time.Minute)) { - return bytesRead, nil + return } return -1, ErrorNotFound } +// read fills in Needle content by looking up n.Id from NeedleMapper +func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) (err error) { + v.dataFileAccessLock.RLock() + defer v.dataFileAccessLock.RUnlock() + + nv, ok := v.nm.Get(n.Id) + if !ok || nv.Offset.IsZero() { + return ErrorNotFound + } + readSize := nv.Size + if readSize.IsDeleted() { + if readOption != nil && readOption.ReadDeleted && readSize != TombstoneFileSize { + glog.V(3).Infof("reading deleted %s", n.String()) + readSize = -readSize + } else { + return ErrorDeleted + } + } + if readSize == 0 { + return nil + } + + if readOption.VolumeRevision != v.SuperBlock.CompactionRevision { + // the volume is compacted + readOption.IsOutOfRange = false + err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) + } + buf := mem.Allocate(1024 * 1024) + defer mem.Free(buf) + actualOffset := nv.Offset.ToActualOffset() + if readOption.IsOutOfRange { + actualOffset += int64(MaxPossibleVolumeSize) + } + + return n.ReadNeedleDataInto(v.DataBackend, actualOffset, buf, writer, offset, size) +} + // read fills in Needle content by looking up n.Id from NeedleMapper func (v *Volume) ReadNeedleBlob(offset int64, size Size) ([]byte, error) { v.dataFileAccessLock.RLock() diff --git a/weed/storage/volume_read_all.go b/weed/storage/volume_read_all.go new file mode 100644 index 000000000..453a4495c --- /dev/null +++ b/weed/storage/volume_read_all.go @@ -0,0 +1,42 @@ +package storage + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/super_block" +) + +type VolumeFileScanner4ReadAll struct { + Stream volume_server_pb.VolumeServer_ReadAllNeedlesServer + V *Volume +} + +func (scanner *VolumeFileScanner4ReadAll) VisitSuperBlock(superBlock super_block.SuperBlock) error { + return nil + +} +func (scanner *VolumeFileScanner4ReadAll) ReadNeedleBody() bool { + return true +} + +func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error { + + nv, ok := scanner.V.nm.Get(n.Id) + if !ok { + return nil + } + if nv.Offset.ToActualOffset() != offset { + return nil + } + + sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{ + VolumeId: uint32(scanner.V.Id), + NeedleId: uint64(n.Id), + Cookie: uint32(n.Cookie), + NeedleBlob: n.Data, + }) + if sendErr != nil { + return sendErr + } + return nil +} diff --git a/weed/storage/volume_stream_write.go b/weed/storage/volume_stream_write.go index d229bdf20..bd8084a3b 100644 --- a/weed/storage/volume_stream_write.go +++ b/weed/storage/volume_stream_write.go @@ -3,10 +3,11 @@ package storage import ( "bufio" "fmt" - "github.com/chrislusf/seaweedfs/weed/util" "io" "time" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/needle" diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 067456dfa..1ad468958 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -17,6 +17,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +type ProgressFunc func(processed int64) bool + func (v *Volume) garbageLevel() float64 { if v.ContentSize() == 0 { return 0 @@ -62,7 +64,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error } // compact a volume based on deletions in .idx files -func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) error { +func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error { if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory return nil @@ -77,13 +79,16 @@ func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) erro v.lastCompactIndexOffset = v.IndexFileSize() v.lastCompactRevision = v.SuperBlock.CompactionRevision glog.V(3).Infof("creating copies for volume %d ...", v.Id) + if v.DataBackend == nil { + return fmt.Errorf("volume %d backend is empty remote:%v", v.Id, v.HasRemoteFile()) + } if err := v.DataBackend.Sync(); err != nil { glog.V(0).Infof("compact2 fail to sync volume dat %d: %v", v.Id, err) } if err := v.nm.Sync(); err != nil { glog.V(0).Infof("compact2 fail to sync volume idx %d: %v", v.Id, err) } - return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond) + return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond, progressFn) } func (v *Volume) CommitCompact() error { @@ -92,9 +97,9 @@ func (v *Volume) CommitCompact() error { } glog.V(0).Infof("Committing volume %d vacuuming...", v.Id) - v.isCompacting = true + v.isCommitCompacting = true defer func() { - v.isCompacting = false + v.isCommitCompacting = false }() v.dataFileAccessLock.Lock() @@ -287,6 +292,9 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err) } dstDatBackend.Write(needleBytes) + if err := dstDatBackend.Sync(); err != nil { + return fmt.Errorf("cannot sync needle %s: %v", dstDatBackend.File.Name(), err) + } util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize)) } else { //deleted needle //fakeDelNeedle 's default Data field is nil @@ -306,6 +314,12 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI newIdxFileName, err) } _, err = idx.Write(idxEntryBytes) + if err != nil { + return fmt.Errorf("cannot write indexfile %s: %v", newIdxFileName, err) + } + if err := idx.Sync(); err != nil { + return fmt.Errorf("cannot sync indexfile %s: %v", newIdxFileName, err) + } } return nil @@ -359,7 +373,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca dst backend.BackendStorageFile ) if dst, err = backend.CreateVolumeFile(dstName, preallocate, 0); err != nil { - return + return err } defer dst.Close() @@ -375,20 +389,19 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca } err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind, scanner) if err != nil { - return nil + return err } - err = nm.SaveToIdx(idxName) - return + return nm.SaveToIdx(idxName) } -func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate int64, compactionBytePerSecond int64) (err error) { +func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate, compactionBytePerSecond int64, progressFn ProgressFunc) (err error) { var ( srcDatBackend, dstDatBackend backend.BackendStorageFile dataFile *os.File ) if dstDatBackend, err = backend.CreateVolumeFile(dstDatName, preallocate, 0); err != nil { - return + return err } defer dstDatBackend.Close() @@ -397,7 +410,7 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str newNm := needle_map.NewMemDb() defer newNm.Close() if err = oldNm.LoadFromIdx(srcIdxName); err != nil { - return + return err } if dataFile, err = os.Open(srcDatName); err != nil { return err @@ -413,7 +426,7 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str writeThrottler := util.NewWriteThrottler(compactionBytePerSecond) - oldNm.AscendingVisit(func(value needle_map.NeedleValue) error { + err = oldNm.AscendingVisit(func(value needle_map.NeedleValue) error { offset, size := value.Offset, value.Size @@ -421,10 +434,15 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str return nil } + if progressFn != nil { + if !progressFn(offset.ToActualOffset()) { + return fmt.Errorf("interrupted") + } + } + n := new(needle.Needle) - err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version) - if err != nil { - return nil + if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version); err != nil { + return fmt.Errorf("cannot hydrate needle from file: %s", err) } if n.HasTtl() && now >= n.LastModified+uint64(sb.Ttl.Minutes()*60) { @@ -444,8 +462,10 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str return nil }) + if err != nil { + return err + } - newNm.SaveToIdx(datIdxName) + return newNm.SaveToIdx(datIdxName) - return } diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index c9596d11d..8212d86c7 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -1,9 +1,7 @@ package storage import ( - "io/ioutil" "math/rand" - "os" "testing" "time" @@ -45,7 +43,7 @@ preparing test prerequisite easier ) func TestMakeDiff(t *testing.T) { v := new(Volume) - //lastCompactIndexOffset value is the index file size before step 4 + // lastCompactIndexOffset value is the index file size before step 4 v.lastCompactIndexOffset = 96 v.SuperBlock.Version = 0x2 /* @@ -63,11 +61,7 @@ func TestMakeDiff(t *testing.T) { } func TestCompaction(t *testing.T) { - dir, err := ioutil.TempDir("", "example") - if err != nil { - t.Fatalf("temp dir creation: %v", err) - } - defer os.RemoveAll(dir) // clean up + dir := t.TempDir() v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0) if err != nil { @@ -84,7 +78,7 @@ func TestCompaction(t *testing.T) { } startTime := time.Now() - v.Compact2(0, 0) + v.Compact2(0, 0, nil) speed := float64(v.ContentSize()) / time.Now().Sub(startTime).Seconds() t.Logf("compaction speed: %.2f bytes/s", speed) diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index 794b1c125..7a96c9695 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -54,7 +54,7 @@ func (v *Volume) isFileUnchanged(n *needle.Needle) bool { // Destroy removes everything related to this volume func (v *Volume) Destroy() (err error) { - if v.isCompacting { + if v.isCompacting || v.isCommitCompacting { err = fmt.Errorf("volume %d is compacting", v.Id) return } diff --git a/weed/storage/volume_write_test.go b/weed/storage/volume_write_test.go new file mode 100644 index 000000000..11fe49358 --- /dev/null +++ b/weed/storage/volume_write_test.go @@ -0,0 +1,55 @@ +package storage + +import ( + "fmt" + "testing" + "time" + + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/super_block" + "github.com/chrislusf/seaweedfs/weed/storage/types" +) + +func TestSearchVolumesWithDeletedNeedles(t *testing.T) { + dir := t.TempDir() + + v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0) + if err != nil { + t.Fatalf("volume creation: %v", err) + } + + count := 20 + + for i := 1; i < count; i++ { + n := newRandomNeedle(uint64(i)) + _, _, _, err := v.writeNeedle2(n, true, false) + if err != nil { + t.Fatalf("write needle %d: %v", i, err) + } + } + + for i := 1; i < 15; i++ { + n := newEmptyNeedle(uint64(i)) + err := v.nm.Put(n.Id, types.Offset{}, types.TombstoneFileSize) + if err != nil { + t.Fatalf("delete needle %d: %v", i, err) + } + } + + ts1 := time.Now().UnixNano() + + for i := 15; i < count; i++ { + n := newEmptyNeedle(uint64(i)) + _, err := v.doDeleteRequest(n) + if err != nil { + t.Fatalf("delete needle %d: %v", i, err) + } + } + + offset, isLast, err := v.BinarySearchByAppendAtNs(uint64(ts1)) + if err != nil { + t.Fatalf("lookup by ts: %v", err) + } + fmt.Printf("offset: %v, isLast: %v\n", offset.ToActualOffset(), isLast) + +} diff --git a/weed/topology/allocate_volume.go b/weed/topology/allocate_volume.go index 7c7fae683..f21d0fc0a 100644 --- a/weed/topology/allocate_volume.go +++ b/weed/topology/allocate_volume.go @@ -15,9 +15,9 @@ type AllocateVolumeResult struct { func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.VolumeId, option *VolumeGrowOption) error { - return operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, deleteErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{ + _, allocateErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{ VolumeId: uint32(vid), Collection: option.Collection, Replication: option.ReplicaPlacement.String(), @@ -26,7 +26,7 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol MemoryMapMaxSizeMb: option.MemoryMapMaxSizeMb, DiskType: string(option.DiskType), }) - return deleteErr + return allocateErr }) } diff --git a/weed/topology/cluster_commands.go b/weed/topology/cluster_commands.go index 152691ccb..1bcc6b449 100644 --- a/weed/topology/cluster_commands.go +++ b/weed/topology/cluster_commands.go @@ -1,9 +1,12 @@ package topology import ( + "encoding/json" + "fmt" "github.com/chrislusf/raft" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/needle" + hashicorpRaft "github.com/hashicorp/raft" ) type MaxVolumeIdCommand struct { @@ -20,6 +23,7 @@ func (c *MaxVolumeIdCommand) CommandName() string { return "MaxVolumeId" } +// deprecatedCommandApply represents the old interface to apply a command to the server. func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) { topo := server.Context().(*Topology) before := topo.GetMaxVolumeId() @@ -29,3 +33,19 @@ func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) { return nil, nil } + +func (s *MaxVolumeIdCommand) Persist(sink hashicorpRaft.SnapshotSink) error { + b, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + _, err = sink.Write(b) + if err != nil { + sink.Cancel() + return fmt.Errorf("sink.Write(): %v", err) + } + return sink.Close() +} + +func (s *MaxVolumeIdCommand) Release() { +} diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 2813f7b45..6bdbd965f 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -2,22 +2,23 @@ package topology import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" - "strconv" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage" ) type DataNode struct { NodeImpl Ip string Port int + GrpcPort int PublicUrl string LastSeen int64 // unix time in seconds + Counter int // in race condition, the previous dataNode was not dead } func NewDataNode(id string) *DataNode { @@ -109,6 +110,9 @@ func (dn *DataNode) DeltaUpdateVolumes(newVolumes, deletedVolumes []storage.Volu for _, v := range deletedVolumes { disk := dn.getOrCreateDisk(v.DiskType) + if _, found := disk.volumes[v.Id]; !found { + continue + } delete(disk.volumes, v.Id) deltaDiskUsages := newDiskUsages() @@ -206,7 +210,11 @@ func (dn *DataNode) MatchLocation(ip string, port int) bool { } func (dn *DataNode) Url() string { - return dn.Ip + ":" + strconv.Itoa(dn.Port) + return util.JoinHostPort(dn.Ip, dn.Port) +} + +func (dn *DataNode) ServerAddress() pb.ServerAddress { + return pb.NewServerAddress(dn.Ip, dn.Port, dn.GrpcPort) } func (dn *DataNode) ToMap() interface{} { @@ -240,6 +248,7 @@ func (dn *DataNode) ToDataNodeInfo() *master_pb.DataNodeInfo { m := &master_pb.DataNodeInfo{ Id: string(dn.Id()), DiskInfos: make(map[string]*master_pb.DiskInfo), + GrpcPort: uint32(dn.GrpcPort), } for _, c := range dn.Children() { disk := c.(*Disk) diff --git a/weed/topology/data_node_ec.go b/weed/topology/data_node_ec.go index 330b16b24..bf72fa9af 100644 --- a/weed/topology/data_node_ec.go +++ b/weed/topology/data_node_ec.go @@ -58,7 +58,7 @@ func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) } for _, ecShards := range actualShards { - if dn.hasEcShards(ecShards.VolumeId) { + if dn.HasEcShards(ecShards.VolumeId) { continue } @@ -79,7 +79,7 @@ func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) return } -func (dn *DataNode) hasEcShards(volumeId needle.VolumeId) (found bool) { +func (dn *DataNode) HasEcShards(volumeId needle.VolumeId) (found bool) { dn.RLock() defer dn.RUnlock() for _, c := range dn.children { diff --git a/weed/topology/node.go b/weed/topology/node.go index 4772cb411..c5956177a 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -3,6 +3,7 @@ package topology import ( "errors" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" @@ -246,6 +247,14 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi } else if float64(v.Size) > float64(volumeSizeLimit)*growThreshold { n.GetTopology().chanCrowdedVolumes <- v } + copyCount := v.ReplicaPlacement.GetCopyCount() + if copyCount > 1 { + if copyCount > len(n.GetTopology().Lookup(v.Collection, v.Id)) { + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1) + } else { + stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0) + } + } } } } else { diff --git a/weed/topology/rack.go b/weed/topology/rack.go index 8eb2a717c..cd09746b2 100644 --- a/weed/topology/rack.go +++ b/weed/topology/rack.go @@ -3,7 +3,7 @@ package topology import ( "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/types" - "strconv" + "github.com/chrislusf/seaweedfs/weed/util" "time" ) @@ -30,7 +30,7 @@ func (r *Rack) FindDataNode(ip string, port int) *DataNode { } return nil } -func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVolumeCounts map[string]uint32) *DataNode { +func (r *Rack) GetOrCreateDataNode(ip string, port int, grpcPort int, publicUrl string, maxVolumeCounts map[string]uint32) *DataNode { for _, c := range r.Children() { dn := c.(*DataNode) if dn.MatchLocation(ip, port) { @@ -38,9 +38,10 @@ func (r *Rack) GetOrCreateDataNode(ip string, port int, publicUrl string, maxVol return dn } } - dn := NewDataNode(ip + ":" + strconv.Itoa(port)) + dn := NewDataNode(util.JoinHostPort(ip, port)) dn.Ip = ip dn.Port = port + dn.GrpcPort = grpcPort dn.PublicUrl = publicUrl dn.LastSeen = time.Now().Unix() r.LinkChildNode(dn) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index b114b468d..7bb10f1da 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -13,6 +13,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" @@ -28,7 +29,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt var remoteLocations []operation.Location if r.FormValue("type") != "replicate" { // this is the initial request - remoteLocations, err = getWritableRemoteReplications(s, grpcDialOption, volumeId, masterFn) + remoteLocations, err = GetWritableRemoteReplications(s, grpcDialOption, volumeId, masterFn) if err != nil { glog.V(0).Infoln(err) return @@ -44,6 +45,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt if s.GetVolume(volumeId) != nil { isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) if err != nil { + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) glog.V(0).Infoln(err) return @@ -74,6 +76,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt tmpMap := make(map[string]string) err := json.Unmarshal(n.Pairs, &tmpMap) if err != nil { + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc() glog.V(0).Infoln("Unmarshal pairs error:", err) } for k, v := range tmpMap { @@ -83,11 +86,22 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt // volume server do not know about encryption // TODO optimize here to compress data only once - _, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsCompressed(), string(n.Mime), pairMap, jwt) + uploadOption := &operation.UploadOption{ + UploadUrl: u.String(), + Filename: string(n.Name), + Cipher: false, + IsInputCompressed: n.IsCompressed(), + MimeType: string(n.Mime), + PairMap: pairMap, + Jwt: jwt, + } + _, err := operation.UploadData(n.Data, uploadOption) return err }); err != nil { + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) glog.V(0).Infoln(err) + return false, err } } return @@ -100,7 +114,7 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp var remoteLocations []operation.Location if r.FormValue("type") != "replicate" { - remoteLocations, err = getWritableRemoteReplications(store, grpcDialOption, volumeId, masterFn) + remoteLocations, err = GetWritableRemoteReplications(store, grpcDialOption, volumeId, masterFn) if err != nil { glog.V(0).Infoln(err) return @@ -160,7 +174,7 @@ func DistributedOperation(locations []operation.Location, op func(location opera return ret.Error() } -func getWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn) (remoteLocations []operation.Location, err error) { +func GetWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn) (remoteLocations []operation.Location, err error) { v := s.GetVolume(volumeId) if v != nil && v.ReplicaPlacement.GetCopyCount() == 1 { @@ -170,7 +184,7 @@ func getWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOpt // not on local store, or has replications lookupResult, lookupErr := operation.LookupVolumeId(masterFn, grpcDialOption, volumeId.String()) if lookupErr == nil { - selfUrl := s.Ip + ":" + strconv.Itoa(s.Port) + selfUrl := util.JoinHostPort(s.Ip, s.Port) for _, location := range lookupResult.Locations { if location.Url != selfUrl { remoteLocations = append(remoteLocations, location) diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 4cbe22a42..631c1fa29 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -1,14 +1,18 @@ package topology import ( + "encoding/json" "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/storage/types" "math/rand" "sync" "time" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/raft" + hashicorpRaft "github.com/hashicorp/raft" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -39,7 +43,10 @@ type Topology struct { Configuration *Configuration - RaftServer raft.Server + RaftServer raft.Server + HashicorpRaft *hashicorpRaft.Raft + UuidAccessLock sync.RWMutex + UuidMap map[string][]string } func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology { @@ -71,19 +78,25 @@ func (t *Topology) IsLeader() bool { return true } if leader, err := t.Leader(); err == nil { - if t.RaftServer.Name() == leader { + if pb.ServerAddress(t.RaftServer.Name()) == leader { return true } } + } else if t.HashicorpRaft != nil { + if t.HashicorpRaft.State() == hashicorpRaft.Leader { + return true + } } return false } -func (t *Topology) Leader() (string, error) { - l := "" +func (t *Topology) Leader() (pb.ServerAddress, error) { + var l pb.ServerAddress for count := 0; count < 3; count++ { if t.RaftServer != nil { - l = t.RaftServer.Leader() + l = pb.ServerAddress(t.RaftServer.Leader()) + } else if t.HashicorpRaft != nil { + l = pb.ServerAddress(t.HashicorpRaft.Leader()) } else { return "", errors.New("Raft Server not ready yet!") } @@ -123,8 +136,18 @@ func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []* func (t *Topology) NextVolumeId() (needle.VolumeId, error) { vid := t.GetMaxVolumeId() next := vid.Next() - if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next)); err != nil { - return 0, err + if t.RaftServer != nil { + if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next)); err != nil { + return 0, err + } + } else if t.HashicorpRaft != nil { + b, err := json.Marshal(NewMaxVolumeIdCommand(next)) + if err != nil { + return 0, fmt.Errorf("failed marshal NewMaxVolumeIdCommand: %+v", err) + } + if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil { + return 0, future.Error() + } } return next, nil } @@ -136,7 +159,7 @@ func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool { return active > 0 } -func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, uint64, *DataNode, error) { +func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, uint64, *VolumeLocationList, error) { vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType).PickForWrite(count, option) if err != nil { return "", 0, nil, fmt.Errorf("failed to find writable volumes for collection:%s replication:%s ttl:%s error: %v", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String(), err) @@ -145,7 +168,7 @@ func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, return "", 0, nil, fmt.Errorf("no writable volumes available for collection:%s replication:%s ttl:%s", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String()) } fileId := t.Sequence.NextFileId(count) - return needle.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes.Head(), nil + return needle.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes, nil } func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout { @@ -205,7 +228,7 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { vl.EnsureCorrectWritables(&v) } func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { - glog.Infof("removing volume info: %+v", v) + glog.Infof("removing volume info: %+v from %v", v, dn.id) diskType := types.ToDiskType(v.DiskType) volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) volumeLayout.UnRegisterVolume(&v, dn) @@ -282,3 +305,14 @@ func (t *Topology) IncrementalSyncDataNodeRegistration(newVolumes, deletedVolume return } + +func (t *Topology) DataNodeRegistration(dcName, rackName string, dn *DataNode) { + if dn.Parent() != nil { + return + } + // registration to topo + dc := t.GetOrCreateDataCenter(dcName) + rack := dc.GetOrCreateRack(rackName) + rack.LinkChildNode(dn) + glog.Infof("[%s] reLink To topo ", dn.Id()) +} diff --git a/weed/topology/topology_ec.go b/weed/topology/topology_ec.go index 022eeb578..fdc4f274e 100644 --- a/weed/topology/topology_ec.go +++ b/weed/topology/topology_ec.go @@ -2,6 +2,7 @@ package topology import ( "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -135,16 +136,16 @@ func (t *Topology) LookupEcShards(vid needle.VolumeId) (locations *EcShardLocati return } -func (t *Topology) ListEcServersByCollection(collection string) (dataNodes []string) { +func (t *Topology) ListEcServersByCollection(collection string) (dataNodes []pb.ServerAddress) { t.ecShardMapLock.RLock() defer t.ecShardMapLock.RUnlock() - dateNodeMap := make(map[string]bool) + dateNodeMap := make(map[pb.ServerAddress]bool) for _, ecVolumeLocation := range t.ecShardMap { if ecVolumeLocation.Collection == collection { for _, locations := range ecVolumeLocation.Locations { for _, loc := range locations { - dateNodeMap[string(loc.Id())] = true + dateNodeMap[loc.ServerAddress()] = true } } } diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go index 0f1db74df..fe3717233 100644 --- a/weed/topology/topology_event_handling.go +++ b/weed/topology/topology_event_handling.go @@ -1,6 +1,7 @@ package topology import ( + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/types" "google.golang.org/grpc" "math/rand" @@ -24,7 +25,7 @@ func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, g c := time.Tick(15 * time.Minute) for _ = range c { if t.IsLeader() { - t.Vacuum(grpcDialOption, garbageThreshold, preallocate) + t.Vacuum(grpcDialOption, garbageThreshold, 0, "", preallocate) } } }(garbageThreshold) @@ -84,7 +85,8 @@ func (t *Topology) UnRegisterDataNode(dn *DataNode) { negativeUsages := dn.GetDiskUsages().negative() dn.UpAdjustDiskUsageDelta(negativeUsages) - + dn.DeltaUpdateVolumes([]storage.VolumeInfo{}, dn.GetVolumes()) + dn.DeltaUpdateEcShards([]*erasure_coding.EcVolumeInfo{}, dn.GetEcShards()) if dn.Parent() != nil { dn.Parent().UnlinkChildNode(dn.Id()) } diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.go index ecfe9d8d1..2ece48a95 100644 --- a/weed/topology/topology_test.go +++ b/weed/topology/topology_test.go @@ -31,7 +31,7 @@ func TestHandlingVolumeServerHeartbeat(t *testing.T) { maxVolumeCounts := make(map[string]uint32) maxVolumeCounts[""] = 25 maxVolumeCounts["ssd"] = 12 - dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, "127.0.0.1", maxVolumeCounts) + dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", maxVolumeCounts) { volumeCount := 7 @@ -177,7 +177,7 @@ func TestAddRemoveVolume(t *testing.T) { maxVolumeCounts := make(map[string]uint32) maxVolumeCounts[""] = 25 maxVolumeCounts["ssd"] = 12 - dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, "127.0.0.1", maxVolumeCounts) + dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", maxVolumeCounts) v := storage.VolumeInfo{ Id: needle.VolumeId(1), diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 9feb55b73..e53aa2853 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -2,6 +2,8 @@ package topology import ( "context" + "github.com/chrislusf/seaweedfs/weed/pb" + "io" "sync/atomic" "time" @@ -19,8 +21,8 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne ch := make(chan int, locationlist.Length()) errCount := int32(0) for index, dn := range locationlist.list { - go func(index int, url string, vid needle.VolumeId) { - err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + go func(index int, url pb.ServerAddress, vid needle.VolumeId) { + err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{ VolumeId: uint32(vid), }) @@ -39,7 +41,7 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne if err != nil { glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err) } - }(index, dn.Url(), vid) + }(index, dn.ServerAddress(), vid) } vacuumLocationList := NewVolumeLocationList() @@ -58,6 +60,7 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne } return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0 } + func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList, preallocate int64) bool { vl.accessLock.Lock() @@ -66,14 +69,29 @@ func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl * ch := make(chan bool, locationlist.Length()) for index, dn := range locationlist.list { - go func(index int, url string, vid needle.VolumeId) { + go func(index int, url pb.ServerAddress, vid needle.VolumeId) { glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url) - err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{ + err := operation.WithVolumeServerClient(true, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + stream, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{ VolumeId: uint32(vid), Preallocate: preallocate, }) - return err + if err != nil { + return err + } + + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + return recvErr + } + } + glog.V(0).Infof("%d vacuum %d on %s processed %d bytes", index, vid, url, resp.ProcessedBytes) + } + return nil }) if err != nil { glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err) @@ -82,7 +100,7 @@ func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl * glog.V(0).Infof("Complete vacuuming %d on %s", vid, url) ch <- true } - }(index, dn.Url(), vid) + }(index, dn.ServerAddress(), vid) } isVacuumSuccess := true @@ -99,12 +117,13 @@ func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl * } return isVacuumSuccess } + func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool { isCommitSuccess := true isReadOnly := false for _, dn := range locationlist.list { glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url()) - err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{ VolumeId: uint32(vid), }) @@ -127,10 +146,11 @@ func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *V } return isCommitSuccess } + func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) { for _, dn := range locationlist.list { glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url()) - err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{ VolumeId: uint32(vid), }) @@ -144,7 +164,7 @@ func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl * } } -func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) { +func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, volumeId uint32, collection string, preallocate int64) { // if there is vacuum going on, return immediately swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1) @@ -155,12 +175,19 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float // now only one vacuum process going on - glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold) + glog.V(1).Infof("Start vacuum on demand with threshold: %f collection: %s volumeId: %d", + garbageThreshold, collection, volumeId) for _, col := range t.collectionMap.Items() { c := col.(*Collection) + if collection != "" && collection != c.Name { + continue + } for _, vl := range c.storageType2VolumeLayout.Items() { if vl != nil { volumeLayout := vl.(*VolumeLayout) + if volumeId > 0 && volumeLayout.Lookup(needle.VolumeId(volumeId)) == nil { + continue + } t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate) } } diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index e2949848d..238ca99f4 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -3,6 +3,7 @@ package topology import ( "encoding/json" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "math/rand" "sync" @@ -77,42 +78,50 @@ func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) { return } -func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount int) (count int, err error) { +func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount int) (result []*master_pb.VolumeLocation, err error) { if targetCount == 0 { targetCount = vg.findVolumeCount(option.ReplicaPlacement.GetCopyCount()) } - count, err = vg.GrowByCountAndType(grpcDialOption, targetCount, option, topo) - if count > 0 && count%option.ReplicaPlacement.GetCopyCount() == 0 { - return count, nil + result, err = vg.GrowByCountAndType(grpcDialOption, targetCount, option, topo) + if len(result) > 0 && len(result)%option.ReplicaPlacement.GetCopyCount() == 0 { + return result, nil } - return count, err + return result, err } -func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount int, option *VolumeGrowOption, topo *Topology) (counter int, err error) { +func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount int, option *VolumeGrowOption, topo *Topology) (result []*master_pb.VolumeLocation, err error) { vg.accessLock.Lock() defer vg.accessLock.Unlock() for i := 0; i < targetCount; i++ { - if c, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil { - counter += c + if res, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil { + result = append(result, res...) } else { - glog.V(0).Infof("create %d volume, created %d: %v", targetCount, counter, e) - return counter, e + glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e) + return result, e } } return } -func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (int, error) { +func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (result []*master_pb.VolumeLocation, err error) { servers, e := vg.findEmptySlotsForOneVolume(topo, option) if e != nil { - return 0, e + return nil, e } vid, raftErr := topo.NextVolumeId() if raftErr != nil { - return 0, raftErr + return nil, raftErr } - err := vg.grow(grpcDialOption, topo, vid, option, servers...) - return len(servers), err + if err = vg.grow(grpcDialOption, topo, vid, option, servers...); err == nil { + for _, server := range servers { + result = append(result, &master_pb.VolumeLocation{ + Url: server.Url(), + PublicUrl: server.PublicUrl, + NewVids: []uint32{uint32(vid)}, + }) + } + } + return } // 1. find the main data node @@ -181,7 +190,7 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum return nil, rackErr } - //find main rack and other racks + //find main server and other servers mainServer, otherServers, serverErr := mainRack.(*Rack).PickNodesByWeight(rp.SameRackCount+1, option, func(node Node) error { if option.DataNode != "" && node.IsDataNode() && node.Id() != NodeId(option.DataNode) { return fmt.Errorf("Not matching preferred data node:%s", option.DataNode) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index f315cb7e4..167aee8ea 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -114,6 +114,8 @@ type VolumeLayout struct { volumeSizeLimit uint64 replicationAsMin bool accessLock sync.RWMutex + growRequestCount int + growRequestTime time.Time } type VolumeLayoutStats struct { @@ -138,9 +140,7 @@ func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType } func (vl *VolumeLayout) String() string { - vl.accessLock.RLock() - defer vl.accessLock.RUnlock() - return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit) + return fmt.Sprintf("rp:%v, ttl:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.writables, vl.volumeSizeLimit) } func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { @@ -218,6 +218,13 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) { vl.setVolumeWritable(vid) } } else { + if !vl.enoughCopies(vid) { + glog.V(0).Infof("volume %d does not have enough copies", vid) + } + if !vl.isAllWritable(vid) { + glog.V(0).Infof("volume %d are not all writable", vid) + } + glog.V(0).Infof("volume %d remove from writable", vid) vl.removeFromWritable(vid) } } @@ -279,7 +286,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*n //glog.V(0).Infoln("No more writable volumes!") return nil, 0, nil, errors.New("No more writable volumes!") } - if option.DataCenter == "" { + if option.DataCenter == "" && option.Rack == "" && option.DataNode == "" { vid := vl.writables[rand.Intn(lenWriters)] locationList := vl.vid2location[vid] if locationList != nil { @@ -293,23 +300,45 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*n for _, v := range vl.writables { volumeLocationList := vl.vid2location[v] for _, dn := range volumeLocationList.list { - if dn.GetDataCenter().Id() == NodeId(option.DataCenter) { - if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) { - continue - } - if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) { - continue - } - counter++ - if rand.Intn(counter) < 1 { - vid, locationList = v, volumeLocationList - } + if option.DataCenter != "" && dn.GetDataCenter().Id() != NodeId(option.DataCenter) { + continue + } + if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) { + continue + } + if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) { + continue + } + counter++ + if rand.Intn(counter) < 1 { + vid, locationList = v, volumeLocationList.Copy() } } } return &vid, count, locationList, nil } +func (vl *VolumeLayout) HasGrowRequest() bool { + if vl.growRequestCount > 0 && vl.growRequestTime.Add(time.Minute).After(time.Now()) { + return true + } + return false +} +func (vl *VolumeLayout) AddGrowRequest() { + vl.growRequestTime = time.Now() + vl.growRequestCount++ +} +func (vl *VolumeLayout) DoneGrowRequest() { + vl.growRequestTime = time.Unix(0, 0) + vl.growRequestCount = 0 +} + +func (vl *VolumeLayout) ShouldGrowVolumes(option *VolumeGrowOption) bool { + active, crowded := vl.GetActiveVolumeCount(option) + //glog.V(0).Infof("active volume: %d, high usage volume: %d\n", active, high) + return active <= crowded +} + func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (active, crowded int) { vl.accessLock.RLock() defer vl.accessLock.RUnlock() @@ -411,8 +440,11 @@ func (vl *VolumeLayout) SetVolumeCapacityFull(vid needle.VolumeId) bool { vl.accessLock.Lock() defer vl.accessLock.Unlock() - // glog.V(0).Infoln("Volume", vid, "reaches full capacity.") - return vl.removeFromWritable(vid) + wasWritable := vl.removeFromWritable(vid) + if wasWritable { + glog.V(0).Infof("Volume %d reaches full capacity.", vid) + } + return wasWritable } func (vl *VolumeLayout) removeFromCrowded(vid needle.VolumeId) { diff --git a/weed/topology/volume_location_list.go b/weed/topology/volume_location_list.go index 548c4cd25..03580ae5b 100644 --- a/weed/topology/volume_location_list.go +++ b/weed/topology/volume_location_list.go @@ -31,6 +31,11 @@ func (dnll *VolumeLocationList) Head() *DataNode { return dnll.list[0] } +func (dnll *VolumeLocationList) Rest() []*DataNode { + //mark first node as master volume + return dnll.list[1:] +} + func (dnll *VolumeLocationList) Length() int { if dnll == nil { return 0 diff --git a/weed/util/bounded_tree/bounded_tree.go b/weed/util/bounded_tree/bounded_tree.go deleted file mode 100644 index 58911df75..000000000 --- a/weed/util/bounded_tree/bounded_tree.go +++ /dev/null @@ -1,179 +0,0 @@ -package bounded_tree - -import ( - "sync" - - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/util" -) - -type Node struct { - Parent *Node - Name string - Children map[string]*Node -} - -type BoundedTree struct { - root *Node - sync.RWMutex - baseDir util.FullPath -} - -func NewBoundedTree(baseDir util.FullPath) *BoundedTree { - return &BoundedTree{ - root: &Node{ - Name: "/", - }, - baseDir: baseDir, - } -} - -type VisitNodeFunc func(path util.FullPath) (childDirectories []string, err error) - -// If the path is not visited, call the visitFn for each level of directory -// No action if the directory has been visited before or does not exist. -// A leaf node, which has no children, represents a directory not visited. -// A non-leaf node or a non-existing node represents a directory already visited, or does not need to visit. -func (t *BoundedTree) EnsureVisited(p util.FullPath, visitFn VisitNodeFunc) (visitErr error) { - t.Lock() - defer t.Unlock() - - if t.root == nil { - return - } - components := p.Split() - // fmt.Printf("components %v %d\n", components, len(components)) - canDelete, err := t.ensureVisited(t.root, util.FullPath("/"), components, 0, visitFn) - if err != nil { - return err - } - if canDelete { - t.root = nil - } - return nil -} - -func (t *BoundedTree) ensureVisited(n *Node, currentPath util.FullPath, components []string, i int, visitFn VisitNodeFunc) (canDeleteNode bool, visitErr error) { - - // println("ensureVisited", currentPath, i) - - if n == nil { - // fmt.Printf("%s null\n", currentPath) - return - } - - if n.isVisited() { - // fmt.Printf("%s visited %v\n", currentPath, n.Name) - } else { - // fmt.Printf("ensure %v\n", currentPath) - - children, err := visitFn(currentPath) - if err != nil { - glog.V(0).Infof("failed to visit %s: %v", currentPath, err) - return false, err - } - - if len(children) == 0 { - // fmt.Printf(" canDelete %v without children\n", currentPath) - return true, nil - } - - n.Children = make(map[string]*Node) - for _, child := range children { - // fmt.Printf(" add child %v %v\n", currentPath, child) - n.Children[child] = &Node{ - Name: child, - } - } - } - - if i >= len(components) { - return - } - - // fmt.Printf(" check child %v %v\n", currentPath, components[i]) - - toVisitNode, found := n.Children[components[i]] - if !found { - // fmt.Printf(" did not find child %v %v\n", currentPath, components[i]) - return - } - - // fmt.Printf(" ensureVisited %v %v\n", currentPath, toVisitNode.Name) - canDelete, childVisitErr := t.ensureVisited(toVisitNode, currentPath.Child(components[i]), components, i+1, visitFn) - if childVisitErr != nil { - return false, childVisitErr - } - if canDelete { - - // fmt.Printf(" delete %v %v\n", currentPath, components[i]) - delete(n.Children, components[i]) - - if len(n.Children) == 0 { - // fmt.Printf(" canDelete %v\n", currentPath) - return true, nil - } - } - - return false, nil - -} - -func (n *Node) isVisited() bool { - if n == nil { - return true - } - if len(n.Children) > 0 { - return true - } - return false -} - -func (n *Node) getChild(childName string) *Node { - if n == nil { - return nil - } - if len(n.Children) > 0 { - return n.Children[childName] - } - return nil -} - -func (t *BoundedTree) HasVisited(p util.FullPath) bool { - - t.RLock() - defer t.RUnlock() - - if t.root == nil { - return true - } - - components := p.Split() - // fmt.Printf("components %v %d\n", components, len(components)) - return t.hasVisited(t.root, util.FullPath("/"), components, 0) -} - -func (t *BoundedTree) hasVisited(n *Node, currentPath util.FullPath, components []string, i int) bool { - - if n == nil { - return true - } - - if !n.isVisited() { - return false - } - - // fmt.Printf(" hasVisited child %v %+v %d\n", currentPath, components, i) - - if i >= len(components) { - return true - } - - toVisitNode, found := n.Children[components[i]] - if !found { - return true - } - - return t.hasVisited(toVisitNode, currentPath.Child(components[i]), components, i+1) - -} diff --git a/weed/util/bounded_tree/bounded_tree_test.go b/weed/util/bounded_tree/bounded_tree_test.go deleted file mode 100644 index 465f1cc9c..000000000 --- a/weed/util/bounded_tree/bounded_tree_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package bounded_tree - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/chrislusf/seaweedfs/weed/util" -) - -var ( - visitFn = func(path util.FullPath) (childDirectories []string, err error) { - fmt.Printf(" visit %v ...\n", path) - switch path { - case "/": - return []string{"a", "g", "h"}, nil - case "/a": - return []string{"b", "f"}, nil - case "/a/b": - return []string{"c", "e"}, nil - case "/a/b/c": - return []string{"d"}, nil - case "/a/b/c/d": - return []string{"i", "j"}, nil - case "/a/b/c/d/i": - return []string{}, nil - case "/a/b/c/d/j": - return []string{}, nil - case "/a/b/e": - return []string{}, nil - case "/a/f": - return []string{}, nil - } - return nil, nil - } - - printMap = func(m map[string]*Node) { - for k := range m { - println(" >", k) - } - } -) - -func TestBoundedTree(t *testing.T) { - - // a/b/c/d/i - // a/b/c/d/j - // a/b/c/d - // a/b/e - // a/f - // g - // h - - tree := NewBoundedTree(util.FullPath("/")) - - tree.EnsureVisited(util.FullPath("/a/b/c"), visitFn) - - assert.Equal(t, true, tree.HasVisited(util.FullPath("/a/b"))) - assert.Equal(t, true, tree.HasVisited(util.FullPath("/a/b/c"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/a/b/c/d"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/a/b/e"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/a/f"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/g"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/h"))) - assert.Equal(t, true, tree.HasVisited(util.FullPath("/"))) - assert.Equal(t, true, tree.HasVisited(util.FullPath("/x"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/a/b/e/x"))) - - printMap(tree.root.Children) - - a := tree.root.getChild("a") - - b := a.getChild("b") - if !b.isVisited() { - t.Errorf("expect visited /a/b") - } - c := b.getChild("c") - if !c.isVisited() { - t.Errorf("expect visited /a/b/c") - } - - d := c.getChild("d") - if d.isVisited() { - t.Errorf("expect unvisited /a/b/c/d") - } - - tree.EnsureVisited(util.FullPath("/a/b/c/d"), visitFn) - tree.EnsureVisited(util.FullPath("/a/b/c/d/i"), visitFn) - tree.EnsureVisited(util.FullPath("/a/b/c/d/j"), visitFn) - tree.EnsureVisited(util.FullPath("/a/b/e"), visitFn) - tree.EnsureVisited(util.FullPath("/a/f"), visitFn) - - printMap(tree.root.Children) - -} - -func TestEmptyBoundedTree(t *testing.T) { - - // g - // h - - tree := NewBoundedTree(util.FullPath("/")) - - visitFn := func(path util.FullPath) (childDirectories []string, err error) { - fmt.Printf(" visit %v ...\n", path) - switch path { - case "/": - return []string{"g", "h"}, nil - } - t.Fatalf("expected visit %s", path) - return nil, nil - } - - tree.EnsureVisited(util.FullPath("/a/b"), visitFn) - - tree.EnsureVisited(util.FullPath("/a/b"), visitFn) - - printMap(tree.root.Children) - - assert.Equal(t, true, tree.HasVisited(util.FullPath("/a/b"))) - assert.Equal(t, true, tree.HasVisited(util.FullPath("/a"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/g"))) - assert.Equal(t, false, tree.HasVisited(util.FullPath("/g/x"))) - -} diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go index 40d24b322..3f3b264b1 100644 --- a/weed/util/chunk_cache/chunk_cache.go +++ b/weed/util/chunk_cache/chunk_cache.go @@ -11,8 +11,7 @@ import ( var ErrorOutOfBounds = errors.New("attempt to read out of bounds") type ChunkCache interface { - GetChunk(fileId string, minSize uint64) (data []byte) - GetChunkSlice(fileId string, offset, length uint64) []byte + ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) SetChunk(fileId string, data []byte) } @@ -44,105 +43,52 @@ func NewTieredChunkCache(maxEntries int64, dir string, diskSizeInUnit int64, uni return c } -func (c *TieredChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) { +func (c *TieredChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) { if c == nil { - return - } - - c.RLock() - defer c.RUnlock() - - return c.doGetChunk(fileId, minSize) -} - -func (c *TieredChunkCache) doGetChunk(fileId string, minSize uint64) (data []byte) { - - if minSize <= c.onDiskCacheSizeLimit0 { - data = c.memCache.GetChunk(fileId) - if len(data) >= int(minSize) { - return data - } - } - - fid, err := needle.ParseFileIdFromString(fileId) - if err != nil { - glog.Errorf("failed to parse file id %s", fileId) - return nil - } - - if minSize <= c.onDiskCacheSizeLimit0 { - data = c.diskCaches[0].getChunk(fid.Key) - if len(data) >= int(minSize) { - return data - } - } - if minSize <= c.onDiskCacheSizeLimit1 { - data = c.diskCaches[1].getChunk(fid.Key) - if len(data) >= int(minSize) { - return data - } - } - { - data = c.diskCaches[2].getChunk(fid.Key) - if len(data) >= int(minSize) { - return data - } - } - - return nil - -} - -func (c *TieredChunkCache) GetChunkSlice(fileId string, offset, length uint64) []byte { - if c == nil { - return nil + return 0, nil } c.RLock() defer c.RUnlock() - return c.doGetChunkSlice(fileId, offset, length) -} - -func (c *TieredChunkCache) doGetChunkSlice(fileId string, offset, length uint64) (data []byte) { - - minSize := offset + length + minSize := offset + uint64(len(data)) if minSize <= c.onDiskCacheSizeLimit0 { - data, err := c.memCache.getChunkSlice(fileId, offset, length) + n, err = c.memCache.readChunkAt(data, fileId, offset) if err != nil { glog.Errorf("failed to read from memcache: %s", err) } - if len(data) >= int(minSize) { - return data + if n >= int(minSize) { + return n, nil } } fid, err := needle.ParseFileIdFromString(fileId) if err != nil { glog.Errorf("failed to parse file id %s", fileId) - return nil + return n, nil } if minSize <= c.onDiskCacheSizeLimit0 { - data = c.diskCaches[0].getChunkSlice(fid.Key, offset, length) - if len(data) >= int(minSize) { - return data + n, err = c.diskCaches[0].readChunkAt(data, fid.Key, offset) + if n >= int(minSize) { + return } } if minSize <= c.onDiskCacheSizeLimit1 { - data = c.diskCaches[1].getChunkSlice(fid.Key, offset, length) - if len(data) >= int(minSize) { - return data + n, err = c.diskCaches[1].readChunkAt(data, fid.Key, offset) + if n >= int(minSize) { + return } } { - data = c.diskCaches[2].getChunkSlice(fid.Key, offset, length) - if len(data) >= int(minSize) { - return data + n, err = c.diskCaches[2].readChunkAt(data, fid.Key, offset) + if n >= int(minSize) { + return } } - return nil + return 0, nil + } func (c *TieredChunkCache) SetChunk(fileId string, data []byte) { diff --git a/weed/util/chunk_cache/chunk_cache_in_memory.go b/weed/util/chunk_cache/chunk_cache_in_memory.go index d725f8a16..2982d0979 100644 --- a/weed/util/chunk_cache/chunk_cache_in_memory.go +++ b/weed/util/chunk_cache/chunk_cache_in_memory.go @@ -1,9 +1,8 @@ package chunk_cache import ( - "time" - "github.com/karlseguin/ccache/v2" + "time" ) // a global cache for recently accessed file chunks @@ -45,6 +44,21 @@ func (c *ChunkCacheInMemory) getChunkSlice(fileId string, offset, length uint64) return data[offset : int(offset)+wanted], nil } +func (c *ChunkCacheInMemory) readChunkAt(buffer []byte, fileId string, offset uint64) (int, error) { + item := c.cache.Get(fileId) + if item == nil { + return 0, nil + } + data := item.Value().([]byte) + item.Extend(time.Hour) + wanted := min(len(buffer), len(data)-int(offset)) + if wanted < 0 { + return 0, ErrorOutOfBounds + } + n := copy(buffer, data[offset:int(offset)+wanted]) + return n, nil +} + func (c *ChunkCacheInMemory) SetChunk(fileId string, data []byte) { localCopy := make([]byte, len(data)) copy(localCopy, data) diff --git a/weed/util/chunk_cache/chunk_cache_on_disk.go b/weed/util/chunk_cache/chunk_cache_on_disk.go index 36de5c972..100b5919e 100644 --- a/weed/util/chunk_cache/chunk_cache_on_disk.go +++ b/weed/util/chunk_cache/chunk_cache_on_disk.go @@ -144,6 +144,28 @@ func (v *ChunkCacheVolume) getNeedleSlice(key types.NeedleId, offset, length uin return data, nil } +func (v *ChunkCacheVolume) readNeedleSliceAt(data []byte, key types.NeedleId, offset uint64) (n int, err error) { + nv, ok := v.nm.Get(key) + if !ok { + return 0, storage.ErrorNotFound + } + wanted := min(len(data), int(nv.Size)-int(offset)) + if wanted < 0 { + // should never happen, but better than panicing + return 0, ErrorOutOfBounds + } + if n, err = v.DataBackend.ReadAt(data, nv.Offset.ToActualOffset()+int64(offset)); err != nil { + return n, fmt.Errorf("read %s.dat [%d,%d): %v", + v.fileName, nv.Offset.ToActualOffset()+int64(offset), int(nv.Offset.ToActualOffset())+int(offset)+wanted, err) + } else { + if n != wanted { + return n, fmt.Errorf("read %d, expected %d", n, wanted) + } + } + + return n, nil +} + func (v *ChunkCacheVolume) WriteNeedle(key types.NeedleId, data []byte) error { offset := v.fileSize diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go index f8325276e..8c7880eee 100644 --- a/weed/util/chunk_cache/chunk_cache_on_disk_test.go +++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go @@ -3,16 +3,13 @@ package chunk_cache import ( "bytes" "fmt" - "io/ioutil" + "github.com/chrislusf/seaweedfs/weed/util/mem" "math/rand" - "os" "testing" ) func TestOnDisk(t *testing.T) { - - tmpDir, _ := ioutil.TempDir("", "c") - defer os.RemoveAll(tmpDir) + tmpDir := t.TempDir() totalDiskSizeInKB := int64(32) @@ -22,7 +19,7 @@ func TestOnDisk(t *testing.T) { type test_data struct { data []byte fileId string - size uint64 + size int } testData := make([]*test_data, writeCount) for i := 0; i < writeCount; i++ { @@ -31,29 +28,35 @@ func TestOnDisk(t *testing.T) { testData[i] = &test_data{ data: buff, fileId: fmt.Sprintf("1,%daabbccdd", i+1), - size: uint64(len(buff)), + size: len(buff), } cache.SetChunk(testData[i].fileId, testData[i].data) // read back right after write - data := cache.GetChunk(testData[i].fileId, testData[i].size) + data := mem.Allocate(testData[i].size) + cache.ReadChunkAt(data, testData[i].fileId, 0) if bytes.Compare(data, testData[i].data) != 0 { t.Errorf("failed to write to and read from cache: %d", i) } + mem.Free(data) } for i := 0; i < 2; i++ { - data := cache.GetChunk(testData[i].fileId, testData[i].size) + data := mem.Allocate(testData[i].size) + cache.ReadChunkAt(data, testData[i].fileId, 0) if bytes.Compare(data, testData[i].data) == 0 { t.Errorf("old cache should have been purged: %d", i) } + mem.Free(data) } for i := 2; i < writeCount; i++ { - data := cache.GetChunk(testData[i].fileId, testData[i].size) + data := mem.Allocate(testData[i].size) + cache.ReadChunkAt(data, testData[i].fileId, 0) if bytes.Compare(data, testData[i].data) != 0 { t.Errorf("failed to write to and read from cache: %d", i) } + mem.Free(data) } cache.Shutdown() @@ -61,10 +64,12 @@ func TestOnDisk(t *testing.T) { cache = NewTieredChunkCache(2, tmpDir, totalDiskSizeInKB, 1024) for i := 0; i < 2; i++ { - data := cache.GetChunk(testData[i].fileId, testData[i].size) + data := mem.Allocate(testData[i].size) + cache.ReadChunkAt(data, testData[i].fileId, 0) if bytes.Compare(data, testData[i].data) == 0 { t.Errorf("old cache should have been purged: %d", i) } + mem.Free(data) } for i := 2; i < writeCount; i++ { @@ -87,10 +92,12 @@ func TestOnDisk(t *testing.T) { */ continue } - data := cache.GetChunk(testData[i].fileId, testData[i].size) + data := mem.Allocate(testData[i].size) + cache.ReadChunkAt(data, testData[i].fileId, 0) if bytes.Compare(data, testData[i].data) != 0 { t.Errorf("failed to write to and read from cache: %d", i) } + mem.Free(data) } cache.Shutdown() diff --git a/weed/util/chunk_cache/on_disk_cache_layer.go b/weed/util/chunk_cache/on_disk_cache_layer.go index b004913ef..de32fb445 100644 --- a/weed/util/chunk_cache/on_disk_cache_layer.go +++ b/weed/util/chunk_cache/on_disk_cache_layer.go @@ -2,12 +2,11 @@ package chunk_cache import ( "fmt" - "path" - "sort" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" + "path" ) type OnDiskCacheLayer struct { @@ -33,10 +32,9 @@ func NewOnDiskCacheLayer(dir, namePrefix string, diskSize int64, segmentCount in } // keep newest cache to the front - sort.Slice(c.diskCaches, func(i, j int) bool { - return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime) + slices.SortFunc(c.diskCaches, func(a, b *ChunkCacheVolume) bool { + return a.lastModTime.After(b.lastModTime) }) - return c } @@ -96,7 +94,7 @@ func (c *OnDiskCacheLayer) getChunkSlice(needleId types.NeedleId, offset, length continue } if err != nil { - glog.Errorf("failed to read cache file %s id %d", diskCache.fileName, needleId) + glog.Warningf("failed to read cache file %s id %d: %v", diskCache.fileName, needleId, err) continue } if len(data) != 0 { @@ -108,6 +106,26 @@ func (c *OnDiskCacheLayer) getChunkSlice(needleId types.NeedleId, offset, length } +func (c *OnDiskCacheLayer) readChunkAt(buffer []byte, needleId types.NeedleId, offset uint64) (n int, err error) { + + for _, diskCache := range c.diskCaches { + n, err = diskCache.readNeedleSliceAt(buffer, needleId, offset) + if err == storage.ErrorNotFound { + continue + } + if err != nil { + glog.Warningf("failed to read cache file %s id %d: %v", diskCache.fileName, needleId, err) + continue + } + if n > 0 { + return + } + } + + return + +} + func (c *OnDiskCacheLayer) shutdown() { for _, diskCache := range c.diskCaches { diff --git a/weed/util/compression_test.go b/weed/util/compression_test.go deleted file mode 100644 index b515e8988..000000000 --- a/weed/util/compression_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package util - -import ( - "testing" - - "golang.org/x/tools/godoc/util" -) - -func TestIsGzippable(t *testing.T) { - buf := make([]byte, 1024) - - isText := util.IsText(buf) - - if isText { - t.Error("buf with zeros are not text") - } - - compressed, _ := GzipData(buf) - - t.Logf("compressed size %d\n", len(compressed)) -} diff --git a/weed/util/config.go b/weed/util/config.go index ae9397340..f09ac7e5e 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -9,6 +9,20 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" ) +var ( + ConfigurationFileDirectory DirectoryValueType +) + +type DirectoryValueType string + +func (s *DirectoryValueType) Set(value string) error { + *s = DirectoryValueType(value) + return nil +} +func (s *DirectoryValueType) String() string { + return string(*s) +} + type Configuration interface { GetString(key string) string GetBool(key string) bool @@ -20,11 +34,12 @@ type Configuration interface { func LoadConfiguration(configFileName string, required bool) (loaded bool) { // find a filer store - viper.SetConfigName(configFileName) // name of config file (without extension) - viper.AddConfigPath(".") // optionally look for config in the working directory - viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths - viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in - viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in + viper.SetConfigName(configFileName) // name of config file (without extension) + viper.AddConfigPath(ResolvePath(ConfigurationFileDirectory.String())) // path to look for the config file in + viper.AddConfigPath(".") // optionally look for config in the working directory + viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths + viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in + viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file if strings.Contains(err.Error(), "Not Found") { diff --git a/weed/util/constants.go b/weed/util/constants.go index d2a90f874..8886e8fd2 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,8 +5,9 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %.02f", sizeLimit, 2.65) - COMMIT = "" + VERSION_NUMBER = fmt.Sprintf("%.02f", 3.12) + VERSION = sizeLimit + " " + VERSION_NUMBER + COMMIT = "" ) func Version() string { diff --git a/weed/util/constants_4bytes.go b/weed/util/constants_4bytes.go index a29d9d3b0..187e33909 100644 --- a/weed/util/constants_4bytes.go +++ b/weed/util/constants_4bytes.go @@ -1,3 +1,4 @@ +//go:build !5BytesOffset // +build !5BytesOffset package util diff --git a/weed/util/constants_5bytes.go b/weed/util/constants_5bytes.go index 91ce4066f..7c6a158cf 100644 --- a/weed/util/constants_5bytes.go +++ b/weed/util/constants_5bytes.go @@ -1,3 +1,4 @@ +//go:build 5BytesOffset // +build 5BytesOffset package util diff --git a/weed/util/file_util.go b/weed/util/file_util.go index f83f80265..6155d18e1 100644 --- a/weed/util/file_util.go +++ b/weed/util/file_util.go @@ -87,3 +87,28 @@ func ResolvePath(path string) string { return path } + +func FileNameBase(filename string) string { + lastDotIndex := strings.LastIndex(filename, ".") + if lastDotIndex < 0 { + return filename + } + return filename[:lastDotIndex] +} + +// Copied from os.WriteFile(), adding file sync. +// see https://github.com/golang/go/issues/20599 +func WriteFile(name string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + if err1 := f.Sync(); err1 != nil && err == nil { + err = err1 + } + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} diff --git a/weed/util/file_util_non_posix.go b/weed/util/file_util_non_posix.go index ffcfef6d5..29aecffa6 100644 --- a/weed/util/file_util_non_posix.go +++ b/weed/util/file_util_non_posix.go @@ -1,3 +1,4 @@ +//go:build linux || darwin || freebsd || netbsd || openbsd || plan9 || solaris || zos // +build linux darwin freebsd netbsd openbsd plan9 solaris zos package util diff --git a/weed/util/file_util_posix.go b/weed/util/file_util_posix.go index 22ca60b3b..0bec8abad 100644 --- a/weed/util/file_util_posix.go +++ b/weed/util/file_util_posix.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package util diff --git a/weed/util/fullpath.go b/weed/util/fullpath.go index 85028b052..f52d4d1d0 100644 --- a/weed/util/fullpath.go +++ b/weed/util/fullpath.go @@ -41,8 +41,11 @@ func (fp FullPath) Child(name string) FullPath { return FullPath(dir + "/" + noPrefix) } -func (fp FullPath) AsInode() uint64 { - return uint64(HashStringToLong(string(fp))) +// AsInode an in-memory only inode representation +func (fp FullPath) AsInode(unixTime int64) uint64 { + inode := uint64(HashStringToLong(string(fp))) + inode = inode + uint64(unixTime)*37 + return inode } // split, but skipping the root diff --git a/weed/util/grace/signal_handling.go b/weed/util/grace/signal_handling.go index 7cca46764..fc7afcad9 100644 --- a/weed/util/grace/signal_handling.go +++ b/weed/util/grace/signal_handling.go @@ -1,3 +1,4 @@ +//go:build !plan9 // +build !plan9 package grace diff --git a/weed/util/grace/signal_handling_notsupported.go b/weed/util/grace/signal_handling_notsupported.go index 5335915a1..de898159a 100644 --- a/weed/util/grace/signal_handling_notsupported.go +++ b/weed/util/grace/signal_handling_notsupported.go @@ -1,3 +1,4 @@ +//go:build plan9 // +build plan9 package grace diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 2efd6b5aa..2f42d3768 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/util/mem" "io" - "io/ioutil" "net/http" "net/url" "strings" @@ -35,7 +35,7 @@ func Post(url string, values url.Values) ([]byte, error) { return nil, err } defer r.Body.Close() - b, err := ioutil.ReadAll(r.Body) + b, err := io.ReadAll(r.Body) if r.StatusCode >= 400 { if err != nil { return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b)) @@ -71,7 +71,7 @@ func Get(url string) ([]byte, bool, error) { reader = response.Body } - b, err := ioutil.ReadAll(reader) + b, err := io.ReadAll(reader) if response.StatusCode >= 400 { retryable := response.StatusCode >= 500 return nil, retryable, fmt.Errorf("%s: %s", url, response.Status) @@ -107,7 +107,7 @@ func Delete(url string, jwt string) error { return e } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return err } @@ -137,7 +137,7 @@ func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err err return } defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) if err != nil { return } @@ -181,7 +181,16 @@ func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) e } func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error) { - response, err := client.Get(fileUrl) + req, err := http.NewRequest("GET", fileUrl, nil) + if err != nil { + return "", nil, nil, err + } + + if len(jwt) > 0 { + req.Header.Set("Authorization", "BEARER "+jwt) + } + + response, err := client.Do(req) if err != nil { return "", nil, nil, err } @@ -271,7 +280,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC } } // drains the response body to avoid memory leak - data, _ := ioutil.ReadAll(reader) + data, _ := io.ReadAll(reader) if len(data) != 0 { glog.V(1).Infof("%s reader has remaining %d bytes", contentEncoding, len(data)) } @@ -318,7 +327,8 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is var ( m int ) - buf := make([]byte, 64*1024) + buf := mem.Allocate(64 * 1024) + defer mem.Free(buf) for { m, err = reader.Read(buf) @@ -327,7 +337,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is return false, nil } if err != nil { - return false, err + return true, err } } @@ -359,7 +369,7 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool return false, nil } -func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { +func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { @@ -371,6 +381,10 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e req.Header.Add("Accept-Encoding", "gzip") } + if len(jwt) > 0 { + req.Header.Set("Authorization", "BEARER "+jwt) + } + r, err := client.Do(req) if err != nil { return nil, err @@ -393,11 +407,30 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e } func CloseResponse(resp *http.Response) { - io.Copy(ioutil.Discard, resp.Body) + reader := &CountingReader{reader: resp.Body} + io.Copy(io.Discard, reader) resp.Body.Close() + if reader.BytesRead > 0 { + glog.V(1).Infof("response leftover %d bytes", reader.BytesRead) + } } func CloseRequest(req *http.Request) { - io.Copy(ioutil.Discard, req.Body) + reader := &CountingReader{reader: req.Body} + io.Copy(io.Discard, reader) req.Body.Close() + if reader.BytesRead > 0 { + glog.V(1).Infof("request leftover %d bytes", reader.BytesRead) + } +} + +type CountingReader struct { + reader io.Reader + BytesRead int +} + +func (r *CountingReader) Read(p []byte) (n int, err error) { + n, err = r.reader.Read(p) + r.BytesRead += n + return n, err } diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index c2158e7eb..422575193 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -56,11 +56,15 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn func(startTi return lb } -func (m *LogBuffer) AddToBuffer(partitionKey, data []byte, eventTsNs int64) { +func (m *LogBuffer) AddToBuffer(partitionKey, data []byte, processingTsNs int64) { + var toFlush *dataToFlush m.Lock() defer func() { m.Unlock() + if toFlush != nil { + m.flushChan <- toFlush + } if m.notifyFn != nil { m.notifyFn() } @@ -68,20 +72,20 @@ func (m *LogBuffer) AddToBuffer(partitionKey, data []byte, eventTsNs int64) { // need to put the timestamp inside the lock var ts time.Time - if eventTsNs == 0 { + if processingTsNs == 0 { ts = time.Now() - eventTsNs = ts.UnixNano() + processingTsNs = ts.UnixNano() } else { - ts = time.Unix(0, eventTsNs) + ts = time.Unix(0, processingTsNs) } - if m.lastTsNs >= eventTsNs { + if m.lastTsNs >= processingTsNs { // this is unlikely to happen, but just in case - eventTsNs = m.lastTsNs + 1 - ts = time.Unix(0, eventTsNs) + processingTsNs = m.lastTsNs + 1 + ts = time.Unix(0, processingTsNs) } - m.lastTsNs = eventTsNs + m.lastTsNs = processingTsNs logEntry := &filer_pb.LogEntry{ - TsNs: eventTsNs, + TsNs: processingTsNs, PartitionKeyHash: util.HashToInt32(partitionKey), Data: data, } @@ -96,7 +100,7 @@ func (m *LogBuffer) AddToBuffer(partitionKey, data []byte, eventTsNs int64) { if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 { // glog.V(4).Infof("%s copyToFlush1 start time %v, ts %v, remaining %d bytes", m.name, m.startTime, ts, len(m.buf)-m.pos) - m.flushChan <- m.copyToFlush() + toFlush = m.copyToFlush() m.startTime = ts if len(m.buf) < size+4 { m.buf = make([]byte, 2*size+4) @@ -148,8 +152,10 @@ func (m *LogBuffer) loopInterval() { return } toFlush := m.copyToFlush() - m.flushChan <- toFlush m.Unlock() + if toFlush != nil { + m.flushChan <- toFlush + } } } @@ -188,16 +194,34 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu m.RLock() defer m.RUnlock() - if !m.lastFlushTime.IsZero() && m.lastFlushTime.After(lastReadTime) { - return nil, ResumeFromDiskError + // Read from disk and memory + // 1. read from disk, last time is = td + // 2. in memory, the earliest time = tm + // if tm <= td, case 2.1 + // read from memory + // if tm is empty, case 2.2 + // read from memory + // if td < tm, case 2.3 + // read from disk again + var tsMemory time.Time + if !m.startTime.IsZero() { + tsMemory = m.startTime } - - /* - fmt.Printf("read buffer %p: %v last stop time: [%v,%v], pos %d, entries:%d, prevBufs:%d\n", m, lastReadTime, m.startTime, m.stopTime, m.pos, len(m.idx), len(m.prevBuffers.buffers)) - for i, prevBuf := range m.prevBuffers.buffers { - fmt.Printf(" prev %d : %s\n", i, prevBuf.String()) + for _, prevBuf := range m.prevBuffers.buffers { + if !prevBuf.startTime.IsZero() && prevBuf.startTime.Before(tsMemory) { + tsMemory = prevBuf.startTime } - */ + } + if tsMemory.IsZero() { // case 2.2 + return nil, nil + } else if lastReadTime.Before(tsMemory) { // case 2.3 + if !m.lastFlushTime.IsZero() { + glog.V(0).Infof("resume with last flush time: %v", m.lastFlushTime) + return nil, ResumeFromDiskError + } + } + + // the following is case 2.1 if lastReadTime.Equal(m.stopTime) { return nil, nil diff --git a/weed/util/log_buffer/log_buffer_test.go b/weed/util/log_buffer/log_buffer_test.go index 7dcfe5f52..915b93bf4 100644 --- a/weed/util/log_buffer/log_buffer_test.go +++ b/weed/util/log_buffer/log_buffer_test.go @@ -27,7 +27,7 @@ func TestNewLogBufferFirstBuffer(t *testing.T) { } receivedmessageCount := 0 - lb.LoopProcessLogData("test", startTime, func() bool { + lb.LoopProcessLogData("test", startTime, 0, func() bool { // stop if no more messages return false }, func(logEntry *filer_pb.LogEntry) error { diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index 02f5af274..99532b47b 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -17,10 +17,11 @@ var ( ResumeFromDiskError = fmt.Errorf("resumeFromDisk") ) -func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startTreadTime time.Time, waitForDataFn func() bool, eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (lastReadTime time.Time, err error) { +func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startReadTime time.Time, stopTsNs int64, + waitForDataFn func() bool, eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (lastReadTime time.Time, isDone bool, err error) { // loop through all messages var bytesBuf *bytes.Buffer - lastReadTime = startTreadTime + lastReadTime = startReadTime defer func() { if bytesBuf != nil { logBuffer.ReleaseMemory(bytesBuf) @@ -34,10 +35,15 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startTreadTime } bytesBuf, err = logBuffer.ReadFromBuffer(lastReadTime) if err == ResumeFromDiskError { - return lastReadTime, ResumeFromDiskError + time.Sleep(1127 * time.Millisecond) + return lastReadTime, isDone, ResumeFromDiskError } // glog.V(4).Infof("%s ReadFromBuffer by %v", readerName, lastReadTime) if bytesBuf == nil { + if stopTsNs != 0 { + isDone = true + return + } if waitForDataFn() { continue } else { @@ -49,7 +55,6 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startTreadTime // fmt.Printf("ReadFromBuffer %s by %v size %d\n", readerName, lastReadTime, len(buf)) batchSize := 0 - var startReadTime time.Time for pos := 0; pos+4 < len(buf); { @@ -67,10 +72,11 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startTreadTime pos += 4 + int(size) continue } - lastReadTime = time.Unix(0, logEntry.TsNs) - if startReadTime.IsZero() { - startReadTime = lastReadTime + if stopTsNs != 0 && logEntry.TsNs > stopTsNs { + isDone = true + return } + lastReadTime = time.Unix(0, logEntry.TsNs) if err = eachLogDataFn(logEntry); err != nil { return diff --git a/weed/util/mem/slot_pool.go b/weed/util/mem/slot_pool.go new file mode 100644 index 000000000..b3493febf --- /dev/null +++ b/weed/util/mem/slot_pool.go @@ -0,0 +1,63 @@ +package mem + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "sync" + "sync/atomic" +) + +var pools []*sync.Pool + +const ( + min_size = 1024 +) + +func bitCount(size int) (count int) { + for ; size > min_size; count++ { + size = (size + 1) >> 1 + } + return +} + +func init() { + // 1KB ~ 256MB + pools = make([]*sync.Pool, bitCount(1024*1024*256)) + for i := 0; i < len(pools); i++ { + slotSize := 1024 << i + pools[i] = &sync.Pool{ + New: func() interface{} { + buffer := make([]byte, slotSize) + return &buffer + }, + } + } +} + +func getSlotPool(size int) (*sync.Pool, bool) { + index := bitCount(size) + if index >= len(pools) { + return nil, false + } + return pools[index], true +} + +var total int64 + +func Allocate(size int) []byte { + if pool, found := getSlotPool(size); found { + newVal := atomic.AddInt64(&total, 1) + glog.V(4).Infof("++> %d", newVal) + + slab := *pool.Get().(*[]byte) + return slab[:size] + } + return make([]byte, size) +} + +func Free(buf []byte) { + if pool, found := getSlotPool(cap(buf)); found { + newVal := atomic.AddInt64(&total, -1) + glog.V(4).Infof("--> %d", newVal) + pool.Put(&buf) + } +} diff --git a/weed/util/mem/slot_pool_test.go b/weed/util/mem/slot_pool_test.go new file mode 100644 index 000000000..44f9ec004 --- /dev/null +++ b/weed/util/mem/slot_pool_test.go @@ -0,0 +1,48 @@ +package mem + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAllocateFree(t *testing.T) { + buf := Allocate(12) + Free(buf) + if cap(buf) != min_size { + t.Errorf("min size error allocated capacity=%d", cap(buf)) + } + if len(buf) != 12 { + t.Errorf("size error") + } + + buf = Allocate(4883) + Free(buf) + if cap(buf) != 1024< 0 { + nb.key = name + } + nb.names[name] = struct{}{} +} +func (nb *NameBatch) DeleteName(name string) { + delete(nb.names, name) + if nb.key == name { + nb.key = "" + for n := range nb.names { + if nb.key == "" || strings.Compare(nb.key, n) > 0 { + nb.key = n + } + } + } +} +func (nb *NameBatch) ListNames(startFrom string, visitNamesFn func(name string) bool) bool { + var names []string + needFilter := startFrom != "" + for n := range nb.names { + if !needFilter || strings.Compare(n, startFrom) >= 0 { + names = append(names, n) + } + } + slices.SortFunc(names, func(a, b string) bool { + return strings.Compare(a, b) < 0 + }) + for _, n := range names { + if !visitNamesFn(n) { + return false + } + } + return true +} + +func NewNameBatch() *NameBatch { + return &NameBatch{ + names: make(map[string]struct{}), + } +} + +func LoadNameBatch(data []byte) *NameBatch { + t := &NameBatchData{} + if len(data) > 0 { + err := proto.Unmarshal(data, t) + if err != nil { + glog.Errorf("unmarshal into NameBatchData{} : %v", err) + return nil + } + } + nb := NewNameBatch() + for _, n := range t.Names { + name := string(n) + if nb.key == "" || strings.Compare(nb.key, name) > 0 { + nb.key = name + } + nb.names[name] = struct{}{} + } + return nb +} + +func (nb *NameBatch) ToBytes() []byte { + t := &NameBatchData{} + for n := range nb.names { + t.Names = append(t.Names, []byte(n)) + } + data, _ := proto.Marshal(t) + return data +} + +func (nb *NameBatch) SplitBy(name string) (x, y *NameBatch) { + x, y = NewNameBatch(), NewNameBatch() + + for n := range nb.names { + // there should be no equal case though + if strings.Compare(n, name) <= 0 { + x.WriteName(n) + } else { + y.WriteName(n) + } + } + return +} diff --git a/weed/util/skiplist/name_list.go b/weed/util/skiplist/name_list.go new file mode 100644 index 000000000..1d4d2ebae --- /dev/null +++ b/weed/util/skiplist/name_list.go @@ -0,0 +1,326 @@ +package skiplist + +import ( + "bytes" +) + +type NameList struct { + skipList *SkipList + batchSize int +} + +func newNameList(store ListStore, batchSize int) *NameList { + return &NameList{ + skipList: New(store), + batchSize: batchSize, + } +} + +/* +Be reluctant to create new nodes. Try to fit into either previous node or next node. +Prefer to add to previous node. + +There are multiple cases after finding the name for greater or equal node + 1. found and node.Key == name + The node contains a batch with leading key the same as the name + nothing to do + 2. no such node found or node.Key > name + + if no such node found + prevNode = list.LargestNode + + // case 2.1 + if previousNode contains name + nothing to do + + // prefer to add to previous node + if prevNode != nil { + // case 2.2 + if prevNode has capacity + prevNode.add name, and save + return + // case 2.3 + split prevNode by name + } + + // case 2.4 + // merge into next node. Avoid too many nodes if adding data in reverse order. + if nextNode is not nil and nextNode has capacity + delete nextNode.Key + nextNode.Key = name + nextNode.batch.add name + insert nodeNode.Key + return + + // case 2.5 + if prevNode is nil + insert new node with key = name, value = batch{name} + return + +*/ +func (nl *NameList) WriteName(name string) error { + + lookupKey := []byte(name) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + // case 1: the name already exists as one leading key in the batch + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + return nil + } + + if !found { + prevNode, err = nl.skipList.GetLargestNode() + if err != nil { + return err + } + } + + if nextNode != nil && prevNode == nil { + prevNode, err = nl.skipList.LoadElement(nextNode.Prev) + if err != nil { + return err + } + } + + if prevNode != nil { + prevNameBatch := LoadNameBatch(prevNode.Value) + // case 2.1 + if prevNameBatch.ContainsName(name) { + return nil + } + + // case 2.2 + if len(prevNameBatch.names) < nl.batchSize { + prevNameBatch.WriteName(name) + return nl.skipList.ChangeValue(prevNode, prevNameBatch.ToBytes()) + } + + // case 2.3 + x, y := prevNameBatch.SplitBy(name) + addToX := len(x.names) <= len(y.names) + if len(x.names) != len(prevNameBatch.names) { + if addToX { + x.WriteName(name) + } + if x.key == prevNameBatch.key { + if err := nl.skipList.ChangeValue(prevNode, x.ToBytes()); err != nil { + return err + } + } else { + if _, err := nl.skipList.InsertByKey([]byte(x.key), 0, x.ToBytes()); err != nil { + return err + } + } + } + if len(y.names) != len(prevNameBatch.names) { + if !addToX { + y.WriteName(name) + } + if y.key == prevNameBatch.key { + if err := nl.skipList.ChangeValue(prevNode, y.ToBytes()); err != nil { + return err + } + } else { + if _, err := nl.skipList.InsertByKey([]byte(y.key), 0, y.ToBytes()); err != nil { + return err + } + } + } + return nil + + } + + // case 2.4 + if nextNode != nil { + nextNameBatch := LoadNameBatch(nextNode.Value) + if len(nextNameBatch.names) < nl.batchSize { + if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } + nextNameBatch.WriteName(name) + if _, err := nl.skipList.InsertByKey([]byte(nextNameBatch.key), 0, nextNameBatch.ToBytes()); err != nil { + return err + } + return nil + } + } + + // case 2.5 + // now prevNode is nil + newNameBatch := NewNameBatch() + newNameBatch.WriteName(name) + if _, err := nl.skipList.InsertByKey([]byte(newNameBatch.key), 0, newNameBatch.ToBytes()); err != nil { + return err + } + + return nil +} + +/* +// case 1: exists in nextNode +if nextNode != nil && nextNode.Key == name { + remove from nextNode, update nextNode + // TODO: merge with prevNode if possible? + return +} +if nextNode is nil + prevNode = list.Largestnode +if prevNode == nil and nextNode.Prev != nil + prevNode = load(nextNode.Prev) + +// case 2: does not exist +// case 2.1 +if prevNode == nil { + return +} +// case 2.2 +if prevNameBatch does not contain name { + return +} + +// case 3 +delete from prevNameBatch +if prevNameBatch + nextNode < capacityList + // case 3.1 + merge +else + // case 3.2 + update prevNode + + +*/ +func (nl *NameList) DeleteName(name string) error { + lookupKey := []byte(name) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + + // case 1 + var nextNameBatch *NameBatch + if nextNode != nil { + nextNameBatch = LoadNameBatch(nextNode.Value) + } + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } + nextNameBatch.DeleteName(name) + if len(nextNameBatch.names) > 0 { + if _, err := nl.skipList.InsertByKey([]byte(nextNameBatch.key), 0, nextNameBatch.ToBytes()); err != nil { + return err + } + } + return nil + } + + if !found { + prevNode, err = nl.skipList.GetLargestNode() + if err != nil { + return err + } + } + + if nextNode != nil && prevNode == nil { + prevNode, err = nl.skipList.LoadElement(nextNode.Prev) + if err != nil { + return err + } + } + + // case 2 + if prevNode == nil { + // case 2.1 + return nil + } + prevNameBatch := LoadNameBatch(prevNode.Value) + if !prevNameBatch.ContainsName(name) { + // case 2.2 + return nil + } + + // case 3 + prevNameBatch.DeleteName(name) + if len(prevNameBatch.names) == 0 { + if _, err := nl.skipList.DeleteByKey(prevNode.Key); err != nil { + return err + } + return nil + } + if nextNameBatch != nil && len(nextNameBatch.names)+len(prevNameBatch.names) < nl.batchSize { + // case 3.1 merge nextNode and prevNode + if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil { + return err + } + for nextName := range nextNameBatch.names { + prevNameBatch.WriteName(nextName) + } + return nl.skipList.ChangeValue(prevNode, prevNameBatch.ToBytes()) + } else { + // case 3.2 update prevNode + return nl.skipList.ChangeValue(prevNode, prevNameBatch.ToBytes()) + } + + return nil +} + +func (nl *NameList) ListNames(startFrom string, visitNamesFn func(name string) bool) error { + lookupKey := []byte(startFrom) + prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey) + if err != nil { + return err + } + if found && bytes.Compare(nextNode.Key, lookupKey) == 0 { + prevNode = nil + } + if !found { + prevNode, err = nl.skipList.GetLargestNode() + if err != nil { + return err + } + } + + if prevNode != nil { + prevNameBatch := LoadNameBatch(prevNode.Value) + if !prevNameBatch.ListNames(startFrom, visitNamesFn) { + return nil + } + } + + for nextNode != nil { + nextNameBatch := LoadNameBatch(nextNode.Value) + if !nextNameBatch.ListNames(startFrom, visitNamesFn) { + return nil + } + nextNode, err = nl.skipList.LoadElement(nextNode.Next[0]) + if err != nil { + return err + } + } + + return nil +} + +func (nl *NameList) RemoteAllListElement() error { + + t := nl.skipList + + nodeRef := t.StartLevels[0] + for nodeRef != nil { + node, err := t.LoadElement(nodeRef) + if err != nil { + return err + } + if node == nil { + return nil + } + if err := t.DeleteElement(node); err != nil { + return err + } + nodeRef = node.Next[0] + } + return nil + +} diff --git a/weed/util/skiplist/name_list_serde.go b/weed/util/skiplist/name_list_serde.go new file mode 100644 index 000000000..0a2052e7b --- /dev/null +++ b/weed/util/skiplist/name_list_serde.go @@ -0,0 +1,71 @@ +package skiplist + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/golang/protobuf/proto" +) + +func LoadNameList(data []byte, store ListStore, batchSize int) *NameList { + + nl := &NameList{ + skipList: New(store), + batchSize: batchSize, + } + + if len(data) == 0 { + return nl + } + + message := &SkipListProto{} + if err := proto.Unmarshal(data, message); err != nil { + glog.Errorf("loading skiplist: %v", err) + } + nl.skipList.MaxNewLevel = int(message.MaxNewLevel) + nl.skipList.MaxLevel = int(message.MaxLevel) + for i, ref := range message.StartLevels { + nl.skipList.StartLevels[i] = &SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + } + } + for i, ref := range message.EndLevels { + nl.skipList.EndLevels[i] = &SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + } + } + return nl +} + +func (nl *NameList) HasChanges() bool { + return nl.skipList.HasChanges +} + +func (nl *NameList) ToBytes() []byte { + message := &SkipListProto{} + message.MaxNewLevel = int32(nl.skipList.MaxNewLevel) + message.MaxLevel = int32(nl.skipList.MaxLevel) + for _, ref := range nl.skipList.StartLevels { + if ref == nil { + break + } + message.StartLevels = append(message.StartLevels, &SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + }) + } + for _, ref := range nl.skipList.EndLevels { + if ref == nil { + break + } + message.EndLevels = append(message.EndLevels, &SkipListElementReference{ + ElementPointer: ref.ElementPointer, + Key: ref.Key, + }) + } + data, err := proto.Marshal(message) + if err != nil { + glog.Errorf("marshal skiplist: %v", err) + } + return data +} diff --git a/weed/util/skiplist/name_list_test.go b/weed/util/skiplist/name_list_test.go new file mode 100644 index 000000000..9d539c9ba --- /dev/null +++ b/weed/util/skiplist/name_list_test.go @@ -0,0 +1,73 @@ +package skiplist + +import ( + "math/rand" + "strconv" + "testing" +) + +const ( + maxNameCount = 100 +) + +func String(x int) string { + return strconv.Itoa(x) +} + +func TestNameList(t *testing.T) { + list := newNameList(memStore, 7) + + for i := 0; i < maxNameCount; i++ { + list.WriteName(String(i)) + } + + counter := 0 + list.ListNames("", func(name string) bool { + counter++ + print(name, " ") + return true + }) + if counter != maxNameCount { + t.Fail() + } + + // list.skipList.println() + + deleteBase := 5 + deleteCount := maxNameCount - 3*deleteBase + + for i := deleteBase; i < deleteBase+deleteCount; i++ { + list.DeleteName(String(i)) + } + + counter = 0 + list.ListNames("", func(name string) bool { + counter++ + return true + }) + // list.skipList.println() + if counter != maxNameCount-deleteCount { + t.Fail() + } + + // randomized deletion + list = newNameList(memStore, 7) + // Delete elements at random positions in the list. + rList := rand.Perm(maxN) + for _, i := range rList { + list.WriteName(String(i)) + } + for _, i := range rList { + list.DeleteName(String(i)) + } + counter = 0 + list.ListNames("", func(name string) bool { + counter++ + print(name, " ") + return true + }) + if counter != 0 { + t.Fail() + } + +} diff --git a/weed/util/skiplist/skiplist.go b/weed/util/skiplist/skiplist.go new file mode 100644 index 000000000..21eed4b43 --- /dev/null +++ b/weed/util/skiplist/skiplist.go @@ -0,0 +1,571 @@ +package skiplist + +// adapted from https://github.com/MauriceGit/skiplist/blob/master/skiplist.go + +import ( + "bytes" + "fmt" + "math/bits" + "math/rand" + "time" +) + +const ( + // maxLevel denotes the maximum height of the skiplist. This height will keep the skiplist + // efficient for up to 34m entries. If there is a need for much more, please adjust this constant accordingly. + maxLevel = 25 +) + +type SkipList struct { + StartLevels [maxLevel]*SkipListElementReference + EndLevels [maxLevel]*SkipListElementReference + MaxNewLevel int + MaxLevel int + ListStore ListStore + HasChanges bool + // elementCount int +} + +// NewSeedEps returns a new empty, initialized Skiplist. +// Given a seed, a deterministic height/list behaviour can be achieved. +// Eps is used to compare keys given by the ExtractKey() function on equality. +func NewSeed(seed int64, listStore ListStore) *SkipList { + + // Initialize random number generator. + rand.Seed(seed) + //fmt.Printf("SkipList seed: %v\n", seed) + + list := &SkipList{ + MaxNewLevel: maxLevel, + MaxLevel: 0, + ListStore: listStore, + // elementCount: 0, + } + + return list +} + +// New returns a new empty, initialized Skiplist. +func New(listStore ListStore) *SkipList { + return NewSeed(time.Now().UTC().UnixNano(), listStore) +} + +// IsEmpty checks, if the skiplist is empty. +func (t *SkipList) IsEmpty() bool { + return t.StartLevels[0] == nil +} + +func (t *SkipList) generateLevel(maxLevel int) int { + level := maxLevel - 1 + // First we apply some mask which makes sure that we don't get a level + // above our desired level. Then we find the first set bit. + var x = rand.Uint64() & ((1 << uint(maxLevel-1)) - 1) + zeroes := bits.TrailingZeros64(x) + if zeroes <= maxLevel { + level = zeroes + } + + return level +} + +func (t *SkipList) findEntryIndex(key []byte, minLevel int) int { + // Find good entry point so we don't accidentally skip half the list. + for i := t.MaxLevel; i >= 0; i-- { + if t.StartLevels[i] != nil && bytes.Compare(t.StartLevels[i].Key, key) < 0 || i <= minLevel { + return i + } + } + return 0 +} + +func (t *SkipList) findExtended(key []byte, findGreaterOrEqual bool) (prevElementIfVisited *SkipListElement, foundElem *SkipListElement, ok bool, err error) { + + foundElem = nil + ok = false + + if t.IsEmpty() { + return + } + + index := t.findEntryIndex(key, 0) + var currentNode *SkipListElement + + currentNode, err = t.LoadElement(t.StartLevels[index]) + if err != nil { + return + } + if currentNode == nil { + return + } + + // In case, that our first element is already greater-or-equal! + if findGreaterOrEqual && compareElement(currentNode, key) > 0 { + foundElem = currentNode + ok = true + return + } + + for { + if compareElement(currentNode, key) == 0 { + foundElem = currentNode + ok = true + return + } + + // Which direction are we continuing next time? + if currentNode.Next[index] != nil && bytes.Compare(currentNode.Next[index].Key, key) <= 0 { + // Go right + currentNode, err = t.LoadElement(currentNode.Next[index]) + if err != nil { + return + } + if currentNode == nil { + return + } + } else { + if index > 0 { + + // Early exit + if currentNode.Next[0] != nil && bytes.Compare(currentNode.Next[0].Key, key) == 0 { + prevElementIfVisited = currentNode + var currentNodeNext *SkipListElement + currentNodeNext, err = t.LoadElement(currentNode.Next[0]) + if err != nil { + return + } + if currentNodeNext == nil { + return + } + foundElem = currentNodeNext + ok = true + return + } + // Go down + index-- + } else { + // Element is not found and we reached the bottom. + if findGreaterOrEqual { + foundElem, err = t.LoadElement(currentNode.Next[index]) + if err != nil { + return + } + ok = foundElem != nil + } + + return + } + } + } +} + +// Find tries to find an element in the skiplist based on the key from the given ListElement. +// elem can be used, if ok is true. +// Find runs in approx. O(log(n)) +func (t *SkipList) Find(key []byte) (prevIfVisited *SkipListElement, elem *SkipListElement, ok bool, err error) { + + if t == nil || key == nil { + return + } + + prevIfVisited, elem, ok, err = t.findExtended(key, false) + return +} + +// FindGreaterOrEqual finds the first element, that is greater or equal to the given ListElement e. +// The comparison is done on the keys (So on ExtractKey()). +// FindGreaterOrEqual runs in approx. O(log(n)) +func (t *SkipList) FindGreaterOrEqual(key []byte) (prevIfVisited *SkipListElement, elem *SkipListElement, ok bool, err error) { + + if t == nil || key == nil { + return + } + + prevIfVisited, elem, ok, err = t.findExtended(key, true) + return +} + +// Delete removes an element equal to e from the skiplist, if there is one. +// If there are multiple entries with the same value, Delete will remove one of them +// (Which one will change based on the actual skiplist layout) +// Delete runs in approx. O(log(n)) +func (t *SkipList) DeleteByKey(key []byte) (id int64, err error) { + + if t == nil || t.IsEmpty() || key == nil { + return + } + + index := t.findEntryIndex(key, t.MaxLevel) + + var currentNode *SkipListElement + var nextNode *SkipListElement + + for { + + if currentNode == nil { + nextNode, err = t.LoadElement(t.StartLevels[index]) + } else { + nextNode, err = t.LoadElement(currentNode.Next[index]) + } + if err != nil { + return id, err + } + + // Found and remove! + if nextNode != nil && compareElement(nextNode, key) == 0 { + + if currentNode != nil { + currentNode.Next[index] = nextNode.Next[index] + if err = t.SaveElement(currentNode); err != nil { + return id, err + } + } + + if index == 0 { + if nextNode.Next[index] != nil { + nextNextNode, err := t.LoadElement(nextNode.Next[index]) + if err != nil { + return id, err + } + if nextNextNode != nil { + nextNextNode.Prev = currentNode.Reference() + if err = t.SaveElement(nextNextNode); err != nil { + return id, err + } + } + } + // t.elementCount-- + id = nextNode.Id + if err = t.DeleteElement(nextNode); err != nil { + return id, err + } + } + + // Link from start needs readjustments. + startNextKey := t.StartLevels[index].Key + if compareElement(nextNode, startNextKey) == 0 { + t.HasChanges = true + t.StartLevels[index] = nextNode.Next[index] + // This was our currently highest node! + if t.StartLevels[index] == nil { + t.MaxLevel = index - 1 + } + } + + // Link from end needs readjustments. + if nextNode.Next[index] == nil { + t.EndLevels[index] = currentNode.Reference() + t.HasChanges = true + } + nextNode.Next[index] = nil + } + + if nextNode != nil && compareElement(nextNode, key) < 0 { + // Go right + currentNode = nextNode + } else { + // Go down + index-- + if index < 0 { + break + } + } + } + return +} + +// Insert inserts the given ListElement into the skiplist. +// Insert runs in approx. O(log(n)) +func (t *SkipList) InsertByKey(key []byte, idIfKnown int64, value []byte) (id int64, err error) { + + if t == nil || key == nil { + return + } + + level := t.generateLevel(t.MaxNewLevel) + + // Only grow the height of the skiplist by one at a time! + if level > t.MaxLevel { + level = t.MaxLevel + 1 + t.MaxLevel = level + t.HasChanges = true + } + + id = idIfKnown + if id == 0 { + id = rand.Int63() + } + elem := &SkipListElement{ + Id: id, + Next: make([]*SkipListElementReference, t.MaxNewLevel, t.MaxNewLevel), + Level: int32(level), + Key: key, + Value: value, + } + + // t.elementCount++ + + newFirst := true + newLast := true + if !t.IsEmpty() { + newFirst = compareElement(elem, t.StartLevels[0].Key) < 0 + newLast = compareElement(elem, t.EndLevels[0].Key) > 0 + } + + normallyInserted := false + if !newFirst && !newLast { + + normallyInserted = true + + index := t.findEntryIndex(key, level) + + var currentNode *SkipListElement + var nextNodeRef *SkipListElementReference + + for { + + if currentNode == nil { + nextNodeRef = t.StartLevels[index] + } else { + nextNodeRef = currentNode.Next[index] + } + + var nextNode *SkipListElement + + // Connect node to next + if index <= level && (nextNodeRef == nil || bytes.Compare(nextNodeRef.Key, key) > 0) { + elem.Next[index] = nextNodeRef + if currentNode != nil { + currentNode.Next[index] = elem.Reference() + if err = t.SaveElement(currentNode); err != nil { + return + } + } + if index == 0 { + elem.Prev = currentNode.Reference() + if nextNodeRef != nil { + if nextNode, err = t.LoadElement(nextNodeRef); err != nil { + return + } + if nextNode != nil { + nextNode.Prev = elem.Reference() + if err = t.SaveElement(nextNode); err != nil { + return + } + } + } + } + } + + if nextNodeRef != nil && bytes.Compare(nextNodeRef.Key, key) <= 0 { + // Go right + if nextNode == nil { + // reuse nextNode when index == 0 + if nextNode, err = t.LoadElement(nextNodeRef); err != nil { + return + } + } + currentNode = nextNode + if currentNode == nil { + return + } + } else { + // Go down + index-- + if index < 0 { + break + } + } + } + } + + // Where we have a left-most position that needs to be referenced! + for i := level; i >= 0; i-- { + + didSomething := false + + if newFirst || normallyInserted { + + if t.StartLevels[i] == nil || bytes.Compare(t.StartLevels[i].Key, key) > 0 { + if i == 0 && t.StartLevels[i] != nil { + startLevelElement, err := t.LoadElement(t.StartLevels[i]) + if err != nil { + return id, err + } + if startLevelElement != nil { + startLevelElement.Prev = elem.Reference() + if err = t.SaveElement(startLevelElement); err != nil { + return id, err + } + } + } + elem.Next[i] = t.StartLevels[i] + t.StartLevels[i] = elem.Reference() + t.HasChanges = true + } + + // link the EndLevels to this element! + if elem.Next[i] == nil { + t.EndLevels[i] = elem.Reference() + t.HasChanges = true + } + + didSomething = true + } + + if newLast { + // Places the element after the very last element on this level! + // This is very important, so we are not linking the very first element (newFirst AND newLast) to itself! + if !newFirst { + if t.EndLevels[i] != nil { + endLevelElement, err := t.LoadElement(t.EndLevels[i]) + if err != nil { + return id, err + } + if endLevelElement != nil { + endLevelElement.Next[i] = elem.Reference() + if err = t.SaveElement(endLevelElement); err != nil { + return id, err + } + } + } + if i == 0 { + elem.Prev = t.EndLevels[i] + } + t.EndLevels[i] = elem.Reference() + t.HasChanges = true + } + + // Link the startLevels to this element! + if t.StartLevels[i] == nil || bytes.Compare(t.StartLevels[i].Key, key) > 0 { + t.StartLevels[i] = elem.Reference() + t.HasChanges = true + } + + didSomething = true + } + + if !didSomething { + break + } + } + + if err = t.SaveElement(elem); err != nil { + return id, err + } + return id, nil + +} + +// GetSmallestNode returns the very first/smallest node in the skiplist. +// GetSmallestNode runs in O(1) +func (t *SkipList) GetSmallestNode() (*SkipListElement, error) { + return t.LoadElement(t.StartLevels[0]) +} + +// GetLargestNode returns the very last/largest node in the skiplist. +// GetLargestNode runs in O(1) +func (t *SkipList) GetLargestNode() (*SkipListElement, error) { + return t.LoadElement(t.EndLevels[0]) +} +func (t *SkipList) GetLargestNodeReference() *SkipListElementReference { + return t.EndLevels[0] +} + +// Next returns the next element based on the given node. +// Next will loop around to the first node, if you call it on the last! +func (t *SkipList) Next(e *SkipListElement) (*SkipListElement, error) { + if e.Next[0] == nil { + return t.LoadElement(t.StartLevels[0]) + } + return t.LoadElement(e.Next[0]) +} + +// Prev returns the previous element based on the given node. +// Prev will loop around to the last node, if you call it on the first! +func (t *SkipList) Prev(e *SkipListElement) (*SkipListElement, error) { + if e.Prev == nil { + return t.LoadElement(t.EndLevels[0]) + } + return t.LoadElement(e.Prev) +} + +// ChangeValue can be used to change the actual value of a node in the skiplist +// without the need of Deleting and reinserting the node again. +// Be advised, that ChangeValue only works, if the actual key from ExtractKey() will stay the same! +// ok is an indicator, wether the value is actually changed. +func (t *SkipList) ChangeValue(e *SkipListElement, newValue []byte) (err error) { + // The key needs to stay correct, so this is very important! + e.Value = newValue + return t.SaveElement(e) +} + +// String returns a string format of the skiplist. Useful to get a graphical overview and/or debugging. +func (t *SkipList) println() { + + print("start --> ") + for i, l := range t.StartLevels { + if l == nil { + break + } + if i > 0 { + print(" -> ") + } + next := "---" + if l != nil { + next = string(l.Key) + } + print(fmt.Sprintf("[%v]", next)) + } + println() + + nodeRef := t.StartLevels[0] + for nodeRef != nil { + print(fmt.Sprintf("%v: ", string(nodeRef.Key))) + node, _ := t.LoadElement(nodeRef) + if node == nil { + break + } + for i := 0; i <= int(node.Level); i++ { + + l := node.Next[i] + + next := "---" + if l != nil { + next = string(l.Key) + } + + if i == 0 { + prev := "---" + + if node.Prev != nil { + prev = string(node.Prev.Key) + } + print(fmt.Sprintf("[%v|%v]", prev, next)) + } else { + print(fmt.Sprintf("[%v]", next)) + } + if i < int(node.Level) { + print(" -> ") + } + + } + nodeRef = node.Next[0] + println() + } + + print("end --> ") + for i, l := range t.EndLevels { + if l == nil { + break + } + if i > 0 { + print(" -> ") + } + next := "---" + if l != nil { + next = string(l.Key) + } + print(fmt.Sprintf("[%v]", next)) + } + println() +} diff --git a/weed/util/skiplist/skiplist.pb.go b/weed/util/skiplist/skiplist.pb.go new file mode 100644 index 000000000..adb121bfc --- /dev/null +++ b/weed/util/skiplist/skiplist.pb.go @@ -0,0 +1,438 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.12.3 +// source: skiplist.proto + +package skiplist + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type SkipListProto struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StartLevels []*SkipListElementReference `protobuf:"bytes,1,rep,name=start_levels,json=startLevels,proto3" json:"start_levels,omitempty"` + EndLevels []*SkipListElementReference `protobuf:"bytes,2,rep,name=end_levels,json=endLevels,proto3" json:"end_levels,omitempty"` + MaxNewLevel int32 `protobuf:"varint,3,opt,name=max_new_level,json=maxNewLevel,proto3" json:"max_new_level,omitempty"` + MaxLevel int32 `protobuf:"varint,4,opt,name=max_level,json=maxLevel,proto3" json:"max_level,omitempty"` +} + +func (x *SkipListProto) Reset() { + *x = SkipListProto{} + if protoimpl.UnsafeEnabled { + mi := &file_skiplist_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SkipListProto) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipListProto) ProtoMessage() {} + +func (x *SkipListProto) ProtoReflect() protoreflect.Message { + mi := &file_skiplist_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SkipListProto.ProtoReflect.Descriptor instead. +func (*SkipListProto) Descriptor() ([]byte, []int) { + return file_skiplist_proto_rawDescGZIP(), []int{0} +} + +func (x *SkipListProto) GetStartLevels() []*SkipListElementReference { + if x != nil { + return x.StartLevels + } + return nil +} + +func (x *SkipListProto) GetEndLevels() []*SkipListElementReference { + if x != nil { + return x.EndLevels + } + return nil +} + +func (x *SkipListProto) GetMaxNewLevel() int32 { + if x != nil { + return x.MaxNewLevel + } + return 0 +} + +func (x *SkipListProto) GetMaxLevel() int32 { + if x != nil { + return x.MaxLevel + } + return 0 +} + +type SkipListElementReference struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ElementPointer int64 `protobuf:"varint,1,opt,name=element_pointer,json=elementPointer,proto3" json:"element_pointer,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *SkipListElementReference) Reset() { + *x = SkipListElementReference{} + if protoimpl.UnsafeEnabled { + mi := &file_skiplist_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SkipListElementReference) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipListElementReference) ProtoMessage() {} + +func (x *SkipListElementReference) ProtoReflect() protoreflect.Message { + mi := &file_skiplist_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SkipListElementReference.ProtoReflect.Descriptor instead. +func (*SkipListElementReference) Descriptor() ([]byte, []int) { + return file_skiplist_proto_rawDescGZIP(), []int{1} +} + +func (x *SkipListElementReference) GetElementPointer() int64 { + if x != nil { + return x.ElementPointer + } + return 0 +} + +func (x *SkipListElementReference) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +type SkipListElement struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Next []*SkipListElementReference `protobuf:"bytes,2,rep,name=next,proto3" json:"next,omitempty"` + Level int32 `protobuf:"varint,3,opt,name=level,proto3" json:"level,omitempty"` + Key []byte `protobuf:"bytes,4,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,5,opt,name=value,proto3" json:"value,omitempty"` + Prev *SkipListElementReference `protobuf:"bytes,6,opt,name=prev,proto3" json:"prev,omitempty"` +} + +func (x *SkipListElement) Reset() { + *x = SkipListElement{} + if protoimpl.UnsafeEnabled { + mi := &file_skiplist_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SkipListElement) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipListElement) ProtoMessage() {} + +func (x *SkipListElement) ProtoReflect() protoreflect.Message { + mi := &file_skiplist_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SkipListElement.ProtoReflect.Descriptor instead. +func (*SkipListElement) Descriptor() ([]byte, []int) { + return file_skiplist_proto_rawDescGZIP(), []int{2} +} + +func (x *SkipListElement) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *SkipListElement) GetNext() []*SkipListElementReference { + if x != nil { + return x.Next + } + return nil +} + +func (x *SkipListElement) GetLevel() int32 { + if x != nil { + return x.Level + } + return 0 +} + +func (x *SkipListElement) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +func (x *SkipListElement) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SkipListElement) GetPrev() *SkipListElementReference { + if x != nil { + return x.Prev + } + return nil +} + +type NameBatchData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Names [][]byte `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` +} + +func (x *NameBatchData) Reset() { + *x = NameBatchData{} + if protoimpl.UnsafeEnabled { + mi := &file_skiplist_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NameBatchData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NameBatchData) ProtoMessage() {} + +func (x *NameBatchData) ProtoReflect() protoreflect.Message { + mi := &file_skiplist_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NameBatchData.ProtoReflect.Descriptor instead. +func (*NameBatchData) Descriptor() ([]byte, []int) { + return file_skiplist_proto_rawDescGZIP(), []int{3} +} + +func (x *NameBatchData) GetNames() [][]byte { + if x != nil { + return x.Names + } + return nil +} + +var File_skiplist_proto protoreflect.FileDescriptor + +var file_skiplist_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x08, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xda, 0x01, 0x0a, 0x0d, 0x53, + 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x45, 0x0a, 0x0c, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x53, 0x6b, + 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x73, 0x12, 0x41, 0x0a, 0x0a, 0x65, 0x6e, 0x64, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, + 0x73, 0x74, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x65, 0x6e, 0x64, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x5f, 0x6e, 0x65, + 0x77, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6d, + 0x61, 0x78, 0x4e, 0x65, 0x77, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, + 0x78, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x6d, + 0x61, 0x78, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x55, 0x0a, 0x18, 0x53, 0x6b, 0x69, 0x70, 0x4c, + 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x65, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0xcf, + 0x01, 0x0a, 0x0f, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x36, 0x0a, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x22, 0x2e, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x53, 0x6b, 0x69, 0x70, + 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x52, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, + 0x76, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x70, 0x72, 0x65, 0x76, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, + 0x74, 0x2e, 0x53, 0x6b, 0x69, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6c, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x04, 0x70, 0x72, 0x65, 0x76, + 0x22, 0x25, 0x0a, 0x0d, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, + 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75, + 0x74, 0x69, 0x6c, 0x2f, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_skiplist_proto_rawDescOnce sync.Once + file_skiplist_proto_rawDescData = file_skiplist_proto_rawDesc +) + +func file_skiplist_proto_rawDescGZIP() []byte { + file_skiplist_proto_rawDescOnce.Do(func() { + file_skiplist_proto_rawDescData = protoimpl.X.CompressGZIP(file_skiplist_proto_rawDescData) + }) + return file_skiplist_proto_rawDescData +} + +var file_skiplist_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_skiplist_proto_goTypes = []interface{}{ + (*SkipListProto)(nil), // 0: skiplist.SkipListProto + (*SkipListElementReference)(nil), // 1: skiplist.SkipListElementReference + (*SkipListElement)(nil), // 2: skiplist.SkipListElement + (*NameBatchData)(nil), // 3: skiplist.NameBatchData +} +var file_skiplist_proto_depIdxs = []int32{ + 1, // 0: skiplist.SkipListProto.start_levels:type_name -> skiplist.SkipListElementReference + 1, // 1: skiplist.SkipListProto.end_levels:type_name -> skiplist.SkipListElementReference + 1, // 2: skiplist.SkipListElement.next:type_name -> skiplist.SkipListElementReference + 1, // 3: skiplist.SkipListElement.prev:type_name -> skiplist.SkipListElementReference + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_skiplist_proto_init() } +func file_skiplist_proto_init() { + if File_skiplist_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_skiplist_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SkipListProto); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_skiplist_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SkipListElementReference); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_skiplist_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SkipListElement); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_skiplist_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NameBatchData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_skiplist_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_skiplist_proto_goTypes, + DependencyIndexes: file_skiplist_proto_depIdxs, + MessageInfos: file_skiplist_proto_msgTypes, + }.Build() + File_skiplist_proto = out.File + file_skiplist_proto_rawDesc = nil + file_skiplist_proto_goTypes = nil + file_skiplist_proto_depIdxs = nil +} diff --git a/weed/util/skiplist/skiplist.proto b/weed/util/skiplist/skiplist.proto new file mode 100644 index 000000000..2991ad830 --- /dev/null +++ b/weed/util/skiplist/skiplist.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package skiplist; + +option go_package = "github.com/chrislusf/seaweedfs/weed/util/skiplist"; + +message SkipListProto { + repeated SkipListElementReference start_levels = 1; + repeated SkipListElementReference end_levels = 2; + int32 max_new_level = 3; + int32 max_level = 4; +} + +message SkipListElementReference { + int64 element_pointer = 1; + bytes key = 2; +} + +message SkipListElement { + int64 id = 1; + repeated SkipListElementReference next = 2; + int32 level = 3; + bytes key = 4; + bytes value = 5; + SkipListElementReference prev = 6; +} + +message NameBatchData { + repeated bytes names = 1; +} \ No newline at end of file diff --git a/weed/util/skiplist/skiplist_serde.go b/weed/util/skiplist/skiplist_serde.go new file mode 100644 index 000000000..a735f66fa --- /dev/null +++ b/weed/util/skiplist/skiplist_serde.go @@ -0,0 +1,51 @@ +package skiplist + +import "bytes" + +func compareElement(a *SkipListElement, key []byte) int { + if len(a.Key) == 0 { + return -1 + } + return bytes.Compare(a.Key, key) +} + +func (node *SkipListElement) Reference() *SkipListElementReference { + if node == nil { + return nil + } + return &SkipListElementReference{ + ElementPointer: node.Id, + Key: node.Key, + } +} + +func (t *SkipList) SaveElement(element *SkipListElement) error { + if element == nil { + return nil + } + return t.ListStore.SaveElement(element.Id, element) +} + +func (t *SkipList) DeleteElement(element *SkipListElement) error { + if element == nil { + return nil + } + return t.ListStore.DeleteElement(element.Id) +} + +func (t *SkipList) LoadElement(ref *SkipListElementReference) (*SkipListElement, error) { + if ref.IsNil() { + return nil, nil + } + return t.ListStore.LoadElement(ref.ElementPointer) +} + +func (ref *SkipListElementReference) IsNil() bool { + if ref == nil { + return true + } + if len(ref.Key) == 0 { + return true + } + return false +} diff --git a/weed/util/skiplist/skiplist_test.go b/weed/util/skiplist/skiplist_test.go new file mode 100644 index 000000000..5b36cacbd --- /dev/null +++ b/weed/util/skiplist/skiplist_test.go @@ -0,0 +1,290 @@ +package skiplist + +import ( + "bytes" + "math/rand" + "strconv" + "testing" +) + +const ( + maxN = 10000 +) + +var ( + memStore = newMemStore() +) + +func TestReverseInsert(t *testing.T) { + list := NewSeed(100, memStore) + + list.InsertByKey([]byte("zzz"), 0, []byte("zzz")) + list.DeleteByKey([]byte("zzz")) + + list.InsertByKey([]byte("aaa"), 0, []byte("aaa")) + + if list.IsEmpty() { + t.Fail() + } + +} + +func TestInsertAndFind(t *testing.T) { + + k0 := []byte("0") + var list *SkipList + + var listPointer *SkipList + listPointer.InsertByKey(k0, 0, k0) + if _, _, ok, _ := listPointer.Find(k0); ok { + t.Fail() + } + + list = New(memStore) + if _, _, ok, _ := list.Find(k0); ok { + t.Fail() + } + if !list.IsEmpty() { + t.Fail() + } + + // Test at the beginning of the list. + for i := 0; i < maxN; i++ { + key := []byte(strconv.Itoa(maxN - i)) + list.InsertByKey(key, 0, key) + } + for i := 0; i < maxN; i++ { + key := []byte(strconv.Itoa(maxN - i)) + if _, _, ok, _ := list.Find(key); !ok { + t.Fail() + } + } + + list = New(memStore) + // Test at the end of the list. + for i := 0; i < maxN; i++ { + key := []byte(strconv.Itoa(i)) + list.InsertByKey(key, 0, key) + } + for i := 0; i < maxN; i++ { + key := []byte(strconv.Itoa(i)) + if _, _, ok, _ := list.Find(key); !ok { + t.Fail() + } + } + + list = New(memStore) + // Test at random positions in the list. + rList := rand.Perm(maxN) + for _, e := range rList { + key := []byte(strconv.Itoa(e)) + // println("insert", e) + list.InsertByKey(key, 0, key) + } + for _, e := range rList { + key := []byte(strconv.Itoa(e)) + // println("find", e) + if _, _, ok, _ := list.Find(key); !ok { + t.Fail() + } + } + // println("print list") + list.println() + +} + +func Element(x int) []byte { + return []byte(strconv.Itoa(x)) +} + +func TestDelete(t *testing.T) { + + k0 := []byte("0") + + var list *SkipList + + // Delete on empty list + list.DeleteByKey(k0) + + list = New(memStore) + + list.DeleteByKey(k0) + if !list.IsEmpty() { + t.Fail() + } + + list.InsertByKey(k0, 0, k0) + list.DeleteByKey(k0) + if !list.IsEmpty() { + t.Fail() + } + + // Delete elements at the beginning of the list. + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(i), 0, Element(i)) + } + for i := 0; i < maxN; i++ { + list.DeleteByKey(Element(i)) + } + if !list.IsEmpty() { + t.Fail() + } + + list = New(memStore) + // Delete elements at the end of the list. + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(i), 0, Element(i)) + } + for i := 0; i < maxN; i++ { + list.DeleteByKey(Element(maxN - i - 1)) + } + if !list.IsEmpty() { + t.Fail() + } + + list = New(memStore) + // Delete elements at random positions in the list. + rList := rand.Perm(maxN) + for _, e := range rList { + list.InsertByKey(Element(e), 0, Element(e)) + } + for _, e := range rList { + list.DeleteByKey(Element(e)) + } + if !list.IsEmpty() { + t.Fail() + } +} + +func TestNext(t *testing.T) { + list := New(memStore) + + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(i), 0, Element(i)) + } + + smallest, _ := list.GetSmallestNode() + largest, _ := list.GetLargestNode() + + lastNode := smallest + node := lastNode + for node != largest { + node, _ = list.Next(node) + // Must always be incrementing here! + if bytes.Compare(node.Key, lastNode.Key) <= 0 { + t.Fail() + } + // Next.Prev must always point to itself! + prevNode, _ := list.Prev(node) + nextNode, _ := list.Next(prevNode) + if nextNode != node { + t.Fail() + } + lastNode = node + } + + if nextNode, _ := list.Next(largest); nextNode != smallest { + t.Fail() + } +} + +func TestPrev(t *testing.T) { + list := New(memStore) + + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(i), 0, Element(i)) + } + + smallest, _ := list.GetSmallestNode() + largest, _ := list.GetLargestNode() + + lastNode := largest + node := lastNode + for node != smallest { + node, _ = list.Prev(node) + // Must always be incrementing here! + if bytes.Compare(node.Key, lastNode.Key) >= 0 { + t.Fail() + } + // Next.Prev must always point to itself! + nextNode, _ := list.Next(node) + prevNode, _ := list.Prev(nextNode) + if prevNode != node { + t.Fail() + } + lastNode = node + } + + if prevNode, _ := list.Prev(smallest); prevNode != largest { + t.Fail() + } +} + +func TestFindGreaterOrEqual(t *testing.T) { + + maxNumber := maxN * 100 + + var list *SkipList + var listPointer *SkipList + + // Test on empty list. + if _, _, ok, _ := listPointer.FindGreaterOrEqual(Element(0)); ok { + t.Errorf("found element 0 in an empty list") + } + + list = New(memStore) + + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(rand.Intn(maxNumber)), 0, Element(i)) + } + + for i := 0; i < maxN; i++ { + key := Element(rand.Intn(maxNumber)) + if _, v, ok, _ := list.FindGreaterOrEqual(key); ok { + // if f is v should be bigger than the element before + if v.Prev != nil && bytes.Compare(key, v.Prev.Key) < 0 { + t.Errorf("PrevV: %s\n key: %s\n\n", string(v.Prev.Key), string(key)) + } + // v should be bigger or equal to f + // If we compare directly, we get an equal key with a difference on the 10th decimal point, which fails. + if bytes.Compare(v.Key, key) < 0 { + t.Errorf("v: %s\n key: %s\n\n", string(v.Key), string(key)) + } + } else { + lastNode, _ := list.GetLargestNode() + lastV := lastNode.GetValue() + // It is OK, to fail, as long as f is bigger than the last element. + if bytes.Compare(key, lastV) <= 0 { + t.Errorf("lastV: %s\n key: %s\n\n", string(lastV), string(key)) + } + } + } + +} + +func TestChangeValue(t *testing.T) { + list := New(memStore) + + for i := 0; i < maxN; i++ { + list.InsertByKey(Element(i), 0, []byte("value")) + } + + for i := 0; i < maxN; i++ { + // The key only looks at the int so the string doesn't matter here! + _, f1, ok, _ := list.Find(Element(i)) + if !ok { + t.Fail() + } + err := list.ChangeValue(f1, []byte("different value")) + if err != nil { + t.Fail() + } + _, f2, ok, _ := list.Find(Element(i)) + if !ok { + t.Fail() + } + if bytes.Compare(f2.GetValue(), []byte("different value")) != 0 { + t.Fail() + } + } +} diff --git a/weed/wdclient/exclusive_locks/exclusive_locker.go b/weed/wdclient/exclusive_locks/exclusive_locker.go index 0fa138496..1767ee4a4 100644 --- a/weed/wdclient/exclusive_locks/exclusive_locker.go +++ b/weed/wdclient/exclusive_locks/exclusive_locker.go @@ -14,7 +14,6 @@ const ( RenewInteval = 4 * time.Second SafeRenewInteval = 3 * time.Second InitLockInteval = 1 * time.Second - AdminLockName = "admin" ) type ExclusiveLocker struct { @@ -22,13 +21,17 @@ type ExclusiveLocker struct { lockTsNs int64 isLocking bool masterClient *wdclient.MasterClient + lockName string + message string } -func NewExclusiveLocker(masterClient *wdclient.MasterClient) *ExclusiveLocker { +func NewExclusiveLocker(masterClient *wdclient.MasterClient, lockName string) *ExclusiveLocker { return &ExclusiveLocker{ masterClient: masterClient, + lockName: lockName, } } + func (l *ExclusiveLocker) IsLocking() bool { return l.isLocking } @@ -51,11 +54,11 @@ func (l *ExclusiveLocker) RequestLock(clientName string) { // retry to get the lease for { - if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error { + if err := l.masterClient.WithClient(false, func(client master_pb.SeaweedClient) error { resp, err := client.LeaseAdminToken(ctx, &master_pb.LeaseAdminTokenRequest{ PreviousToken: atomic.LoadInt64(&l.token), PreviousLockTime: atomic.LoadInt64(&l.lockTsNs), - LockName: AdminLockName, + LockName: l.lockName, ClientName: clientName, }) if err == nil { @@ -79,12 +82,13 @@ func (l *ExclusiveLocker) RequestLock(clientName string) { defer cancel2() for l.isLocking { - if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error { + if err := l.masterClient.WithClient(false, func(client master_pb.SeaweedClient) error { resp, err := client.LeaseAdminToken(ctx2, &master_pb.LeaseAdminTokenRequest{ PreviousToken: atomic.LoadInt64(&l.token), PreviousLockTime: atomic.LoadInt64(&l.lockTsNs), - LockName: AdminLockName, + LockName: l.lockName, ClientName: clientName, + Message: l.message, }) if err == nil { atomic.StoreInt64(&l.token, resp.Token) @@ -110,14 +114,18 @@ func (l *ExclusiveLocker) ReleaseLock() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - l.masterClient.WithClient(func(client master_pb.SeaweedClient) error { + l.masterClient.WithClient(false, func(client master_pb.SeaweedClient) error { client.ReleaseAdminToken(ctx, &master_pb.ReleaseAdminTokenRequest{ PreviousToken: atomic.LoadInt64(&l.token), PreviousLockTime: atomic.LoadInt64(&l.lockTsNs), - LockName: AdminLockName, + LockName: l.lockName, }) return nil }) atomic.StoreInt64(&l.token, 0) atomic.StoreInt64(&l.lockTsNs, 0) } + +func (l *ExclusiveLocker) SetMessage(message string) { + l.message = message +} diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index d76ae31e2..def4a9a71 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -2,6 +2,7 @@ package wdclient import ( "context" + "github.com/chrislusf/seaweedfs/weed/stats" "math/rand" "time" @@ -14,32 +15,69 @@ import ( ) type MasterClient struct { + FilerGroup string clientType string - clientHost string - grpcPort uint32 - currentMaster string - masters []string + clientHost pb.ServerAddress + currentMaster pb.ServerAddress + masters map[string]pb.ServerAddress grpcDialOption grpc.DialOption vidMap + + OnPeerUpdate func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) } -func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost string, clientGrpcPort uint32, clientDataCenter string, masters []string) *MasterClient { +func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters map[string]pb.ServerAddress) *MasterClient { return &MasterClient{ + FilerGroup: filerGroup, clientType: clientType, clientHost: clientHost, - grpcPort: clientGrpcPort, masters: masters, grpcDialOption: grpcDialOption, vidMap: newVidMap(clientDataCenter), } } -func (mc *MasterClient) GetMaster() string { +func (mc *MasterClient) GetLookupFileIdFunction() LookupFileIdFunctionType { + return mc.LookupFileIdWithFallback +} + +func (mc *MasterClient) LookupFileIdWithFallback(fileId string) (fullUrls []string, err error) { + fullUrls, err = mc.vidMap.LookupFileId(fileId) + err = pb.WithMasterClient(false, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { + resp, err := client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{ + VolumeOrFileIds: []string{fileId}, + }) + if err != nil { + return err + } + for vid, vidLocation := range resp.VolumeIdLocations { + for _, vidLoc := range vidLocation.Locations { + loc := Location{ + Url: vidLoc.Url, + PublicUrl: vidLoc.PublicUrl, + GrpcPort: int(vidLoc.GrpcPort), + } + mc.vidMap.addLocation(uint32(vid), loc) + fullUrls = append(fullUrls, loc.Url) + } + } + + return nil + }) + return +} + +func (mc *MasterClient) GetMaster() pb.ServerAddress { mc.WaitUntilConnected() return mc.currentMaster } +func (mc *MasterClient) GetMasters() map[string]pb.ServerAddress { + mc.WaitUntilConnected() + return mc.masters +} + func (mc *MasterClient) WaitUntilConnected() { for mc.currentMaster == "" { time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond) @@ -47,19 +85,19 @@ func (mc *MasterClient) WaitUntilConnected() { } func (mc *MasterClient) KeepConnectedToMaster() { - glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters) + glog.V(1).Infof("%s.%s masterClient bootstraps with masters %v", mc.FilerGroup, mc.clientType, mc.masters) for { mc.tryAllMasters() time.Sleep(time.Second) } } -func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress string) (leader string) { +func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) { for _, master := range mc.masters { if master == myMasterAddress { continue } - if grpcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { + if grpcErr := pb.WithMasterClient(false, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) defer cancel() resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{}) @@ -81,7 +119,7 @@ func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress string) (leader } func (mc *MasterClient) tryAllMasters() { - nextHintedLeader := "" + var nextHintedLeader pb.ServerAddress for _, master := range mc.masters { nextHintedLeader = mc.tryConnectToMaster(master) @@ -90,74 +128,126 @@ func (mc *MasterClient) tryAllMasters() { } mc.currentMaster = "" - mc.vidMap = newVidMap("") } } -func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) { - glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master) - gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { - +func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) { + glog.V(1).Infof("%s.%s masterClient Connecting to master %v", mc.FilerGroup, mc.clientType, master) + stats.MasterClientConnectCounter.WithLabelValues("total").Inc() + gprcErr := pb.WithMasterClient(true, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() stream, err := client.KeepConnected(ctx) if err != nil { - glog.V(1).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err) + glog.V(1).Infof("%s.%s masterClient failed to keep connected to %s: %v", mc.FilerGroup, mc.clientType, master, err) + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc() return err } - if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil { - glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err) + if err = stream.Send(&master_pb.KeepConnectedRequest{ + FilerGroup: mc.FilerGroup, + ClientType: mc.clientType, + ClientAddress: string(mc.clientHost), + Version: util.Version(), + }); err != nil { + glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err) + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc() + return err + } + glog.V(1).Infof("%s.%s masterClient Connected to %v", mc.FilerGroup, mc.clientType, master) + + resp, err := stream.Recv() + if err != nil { + glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err) + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc() return err } - glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master) + // check if it is the leader to determine whether to reset the vidMap + if resp.VolumeLocation != nil && resp.VolumeLocation.Leader != "" { + glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader) + nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader) + stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc() + return nil + } + mc.currentMaster = master + mc.vidMap = newVidMap("") for { - volumeLocation, err := stream.Recv() + resp, err := stream.Recv() if err != nil { - glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err) + glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err) + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc() return err } - // maybe the leader is changed - if volumeLocation.Leader != "" { - glog.V(0).Infof("redirected to leader %v", volumeLocation.Leader) - nextHintedLeader = volumeLocation.Leader - return nil + if resp.VolumeLocation != nil { + // maybe the leader is changed + if resp.VolumeLocation.Leader != "" { + glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader) + nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader) + stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc() + return nil + } + + // process new volume location + loc := Location{ + Url: resp.VolumeLocation.Url, + PublicUrl: resp.VolumeLocation.PublicUrl, + DataCenter: resp.VolumeLocation.DataCenter, + GrpcPort: int(resp.VolumeLocation.GrpcPort), + } + for _, newVid := range resp.VolumeLocation.NewVids { + glog.V(1).Infof("%s.%s: %s masterClient adds volume %d", mc.FilerGroup, mc.clientType, loc.Url, newVid) + mc.addLocation(newVid, loc) + } + for _, deletedVid := range resp.VolumeLocation.DeletedVids { + glog.V(1).Infof("%s.%s: %s masterClient removes volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedVid) + mc.deleteLocation(deletedVid, loc) + } + for _, newEcVid := range resp.VolumeLocation.NewEcVids { + glog.V(1).Infof("%s.%s: %s masterClient adds ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, newEcVid) + mc.addEcLocation(newEcVid, loc) + } + for _, deletedEcVid := range resp.VolumeLocation.DeletedEcVids { + glog.V(1).Infof("%s.%s: %s masterClient removes ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedEcVid) + mc.deleteEcLocation(deletedEcVid, loc) + } } - // process new volume location - loc := Location{ - Url: volumeLocation.Url, - PublicUrl: volumeLocation.PublicUrl, - DataCenter: volumeLocation.DataCenter, - } - for _, newVid := range volumeLocation.NewVids { - glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid) - mc.addLocation(newVid, loc) - } - for _, deletedVid := range volumeLocation.DeletedVids { - glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid) - mc.deleteLocation(deletedVid, loc) + if resp.ClusterNodeUpdate != nil { + update := resp.ClusterNodeUpdate + if mc.OnPeerUpdate != nil { + if update.FilerGroup == mc.FilerGroup { + if update.IsAdd { + glog.V(0).Infof("+ %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader) + } else { + glog.V(0).Infof("- %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader) + } + stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc() + mc.OnPeerUpdate(update, time.Now()) + } + } } + } }) if gprcErr != nil { - glog.V(1).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr) + stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc() + glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr) } return } -func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error { +func (mc *MasterClient) WithClient(streamingMode bool, fn func(client master_pb.SeaweedClient) error) error { return util.Retry("master grpc", func() error { for mc.currentMaster == "" { time.Sleep(3 * time.Second) } - return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { + return pb.WithMasterClient(streamingMode, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error { return fn(client) }) }) diff --git a/weed/wdclient/net2/ip.go b/weed/wdclient/net2/ip.go deleted file mode 100644 index ff5e3b24e..000000000 --- a/weed/wdclient/net2/ip.go +++ /dev/null @@ -1,177 +0,0 @@ -package net2 - -import ( - "fmt" - "log" - "net" - "os" - "strings" - "sync" -) - -var myHostname string -var myHostnameOnce sync.Once - -// Like os.Hostname but caches first successful result, making it cheap to call it -// over and over. -// It will also crash whole process if fetching Hostname fails! -func MyHostname() string { - myHostnameOnce.Do(func() { - var err error - myHostname, err = os.Hostname() - if err != nil { - log.Fatal(err) - } - }) - return myHostname -} - -var myIp4 *net.IPAddr -var myIp4Once sync.Once - -// Resolves `MyHostname()` to an Ip4 address. Caches first successful result, making it -// cheap to call it over and over. -// It will also crash whole process if resolving the IP fails! -func MyIp4() *net.IPAddr { - myIp4Once.Do(func() { - var err error - myIp4, err = net.ResolveIPAddr("ip4", MyHostname()) - if err != nil { - log.Fatal(err) - } - }) - return myIp4 -} - -var myIp6 *net.IPAddr -var myIp6Once sync.Once - -// Resolves `MyHostname()` to an Ip6 address. Caches first successful result, making it -// cheap to call it over and over. -// It will also crash whole process if resolving the IP fails! -func MyIp6() *net.IPAddr { - myIp6Once.Do(func() { - var err error - myIp6, err = net.ResolveIPAddr("ip6", MyHostname()) - if err != nil { - log.Fatal(err) - } - }) - return myIp6 -} - -// This returns the list of local ip addresses which other hosts can connect -// to (NOTE: Loopback ip is ignored). -// Also resolves Hostname to an address and adds it to the list too, so -// IPs from /etc/hosts can work too. -func GetLocalIPs() ([]*net.IP, error) { - hostname, err := os.Hostname() - if err != nil { - return nil, fmt.Errorf("Failed to lookup hostname: %v", err) - } - // Resolves IP Address from Hostname, this way overrides in /etc/hosts - // can work too for IP resolution. - ipInfo, err := net.ResolveIPAddr("ip4", hostname) - if err != nil { - return nil, fmt.Errorf("Failed to resolve ip: %v", err) - } - ips := []*net.IP{&ipInfo.IP} - - // TODO(zviad): Is rest of the code really necessary? - addrs, err := net.InterfaceAddrs() - if err != nil { - return nil, fmt.Errorf("Failed to get interface addresses: %v", err) - } - for _, addr := range addrs { - ipnet, ok := addr.(*net.IPNet) - if !ok { - continue - } - - if ipnet.IP.IsLoopback() { - continue - } - - ips = append(ips, &ipnet.IP) - } - return ips, nil -} - -var localhostIPNets []*net.IPNet - -func init() { - for _, mask := range []string{"127.0.0.1/8", "::1/128"} { - _, ipnet, err := net.ParseCIDR(mask) - if err != nil { - panic(err) - } - localhostIPNets = append(localhostIPNets, ipnet) - } -} - -func IsLocalhostIp(ipStr string) bool { - ip := net.ParseIP(ipStr) - if ip == nil { - return false - } - for _, ipnet := range localhostIPNets { - if ipnet.Contains(ip) { - return true - } - } - return false -} - -// Given a host string, return true if the host is an ip (v4/v6) localhost. -func IsLocalhost(host string) bool { - return IsLocalhostIp(host) || - host == "localhost" || - host == "ip6-localhost" || - host == "ipv6-localhost" -} - -// Resolves hostnames in addresses to actual IP4 addresses. Skips all invalid addresses -// and all addresses that can't be resolved. -// `addrs` are assumed to be of form: [":", ...] -// Returns an error in addition to resolved addresses if not all resolutions succeed. -func ResolveIP4s(addrs []string) ([]string, error) { - resolvedAddrs := make([]string, 0, len(addrs)) - var lastErr error - - for _, server := range addrs { - hostPort := strings.Split(server, ":") - if len(hostPort) != 2 { - lastErr = fmt.Errorf("Skipping invalid address: %s", server) - continue - } - - ip, err := net.ResolveIPAddr("ip4", hostPort[0]) - if err != nil { - lastErr = err - continue - } - resolvedAddrs = append(resolvedAddrs, ip.IP.String()+":"+hostPort[1]) - } - return resolvedAddrs, lastErr -} - -func LookupValidAddrs() (map[string]bool, error) { - hostName, err := os.Hostname() - if err != nil { - return nil, err - } - addrs, err := net.LookupHost(hostName) - if err != nil { - return nil, err - } - validAddrs := make(map[string]bool) - validAddrs[hostName] = true - for _, addr := range addrs { - validAddrs[addr] = true - } - // Special case localhost/127.0.0.1 so that this works on devVMs. It should - // have no affect in production. - validAddrs["127.0.0.1"] = true - validAddrs["localhost"] = true - return validAddrs, nil -} diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go index 271baa132..754c77051 100644 --- a/weed/wdclient/vid_map.go +++ b/weed/wdclient/vid_map.go @@ -3,6 +3,8 @@ package wdclient import ( "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/pb" + "math/rand" "strconv" "strings" "sync" @@ -25,20 +27,27 @@ type Location struct { Url string `json:"url,omitempty"` PublicUrl string `json:"publicUrl,omitempty"` DataCenter string `json:"dataCenter,omitempty"` + GrpcPort int `json:"grpcPort,omitempty"` +} + +func (l Location) ServerAddress() pb.ServerAddress { + return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort) } type vidMap struct { sync.RWMutex - vid2Locations map[uint32][]Location - DataCenter string - cursor int32 + vid2Locations map[uint32][]Location + ecVid2Locations map[uint32][]Location + DataCenter string + cursor int32 } func newVidMap(dataCenter string) vidMap { return vidMap{ - vid2Locations: make(map[uint32][]Location), - DataCenter: dataCenter, - cursor: -1, + vid2Locations: make(map[uint32][]Location), + ecVid2Locations: make(map[uint32][]Location), + DataCenter: dataCenter, + cursor: -1, } } @@ -63,20 +72,24 @@ func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err er if !found { return nil, fmt.Errorf("volume %d not found", id) } + var sameDcServers, otherDcServers []string for _, loc := range locations { if vc.DataCenter == "" || loc.DataCenter == "" || vc.DataCenter != loc.DataCenter { - serverUrls = append(serverUrls, loc.Url) + otherDcServers = append(otherDcServers, loc.Url) } else { - serverUrls = append([]string{loc.Url}, serverUrls...) + sameDcServers = append(sameDcServers, loc.Url) } } + rand.Shuffle(len(sameDcServers), func(i, j int) { + sameDcServers[i], sameDcServers[j] = sameDcServers[j], sameDcServers[i] + }) + rand.Shuffle(len(otherDcServers), func(i, j int) { + otherDcServers[i], otherDcServers[j] = otherDcServers[j], otherDcServers[i] + }) + serverUrls = append(sameDcServers, otherDcServers...) return } -func (vc *vidMap) GetLookupFileIdFunction() LookupFileIdFunctionType { - return vc.LookupFileId -} - func (vc *vidMap) LookupFileId(fileId string) (fullUrls []string, err error) { parts := strings.Split(fileId, ",") if len(parts) != 2 { @@ -109,14 +122,22 @@ func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) { vc.RLock() defer vc.RUnlock() + glog.V(4).Infof("~ lookup volume id %d: %+v ec:%+v", vid, vc.vid2Locations, vc.ecVid2Locations) + locations, found = vc.vid2Locations[vid] - return + if found && len(locations) > 0 { + return + } + locations, found = vc.ecVid2Locations[vid] + return locations, found && len(locations) > 0 } func (vc *vidMap) addLocation(vid uint32, location Location) { vc.Lock() defer vc.Unlock() + glog.V(4).Infof("+ volume id %d: %+v", vid, location) + locations, found := vc.vid2Locations[vid] if !found { vc.vid2Locations[vid] = []Location{location} @@ -133,10 +154,34 @@ func (vc *vidMap) addLocation(vid uint32, location Location) { } +func (vc *vidMap) addEcLocation(vid uint32, location Location) { + vc.Lock() + defer vc.Unlock() + + glog.V(4).Infof("+ ec volume id %d: %+v", vid, location) + + locations, found := vc.ecVid2Locations[vid] + if !found { + vc.ecVid2Locations[vid] = []Location{location} + return + } + + for _, loc := range locations { + if loc.Url == location.Url { + return + } + } + + vc.ecVid2Locations[vid] = append(locations, location) + +} + func (vc *vidMap) deleteLocation(vid uint32, location Location) { vc.Lock() defer vc.Unlock() + glog.V(4).Infof("- volume id %d: %+v", vid, location) + locations, found := vc.vid2Locations[vid] if !found { return @@ -150,3 +195,23 @@ func (vc *vidMap) deleteLocation(vid uint32, location Location) { } } + +func (vc *vidMap) deleteEcLocation(vid uint32, location Location) { + vc.Lock() + defer vc.Unlock() + + glog.V(4).Infof("- ec volume id %d: %+v", vid, location) + + locations, found := vc.ecVid2Locations[vid] + if !found { + return + } + + for i, loc := range locations { + if loc.Url == location.Url { + vc.ecVid2Locations[vid] = append(locations[0:i], locations[i+1:]...) + break + } + } + +} diff --git a/weed/weed.go b/weed/weed.go index 068d2077c..36f354f83 100644 --- a/weed/weed.go +++ b/weed/weed.go @@ -4,6 +4,7 @@ import ( "embed" "fmt" weed_server "github.com/chrislusf/seaweedfs/weed/server" + "github.com/chrislusf/seaweedfs/weed/util" flag "github.com/chrislusf/seaweedfs/weed/util/fla9" "io" "io/fs" @@ -40,6 +41,8 @@ var static embed.FS func init() { weed_server.StaticFS, _ = fs.Sub(static, "static") + + flag.Var(&util.ConfigurationFileDirectory, "config_dir", "directory with toml configuration files") } func main() {