Browse Source

Merge branch 'chrislusf-master'

pull/2939/head
guosj 3 years ago
parent
commit
94c702402e
  1. 9
      .github/workflows/binaries_dev.yml
  2. 5
      .github/workflows/binaries_release0.yml
  3. 5
      .github/workflows/binaries_release1.yml
  4. 5
      .github/workflows/binaries_release2.yml
  5. 5
      .github/workflows/binaries_release3.yml
  6. 2
      .github/workflows/container_dev.yml
  7. 2
      .github/workflows/container_latest.yml
  8. 2
      .github/workflows/container_release1.yml
  9. 2
      .github/workflows/container_release2.yml
  10. 2
      .github/workflows/container_release3.yml
  11. 4
      README.md
  12. 2
      docker/Dockerfile.go_build
  13. 2
      docker/Dockerfile.go_build_large
  14. 4
      docker/Dockerfile.rocksdb_large
  15. 3
      docker/Makefile
  16. 89
      docker/compose/local-hashicorp-raft-compose.yml
  17. 4
      docker/compose/local-s3tests-compose.yml
  18. 153
      go.mod
  19. 730
      go.sum
  20. 4
      k8s/helm_charts2/Chart.yaml
  21. 32
      k8s/helm_charts2/templates/filer-statefulset.yaml
  22. 6
      k8s/helm_charts2/values.yaml
  23. 13
      other/java/client/src/main/proto/filer.proto
  24. 2
      other/java/hdfs-over-ftp/pom.xml
  25. 2
      other/java/hdfs3/pom.xml
  26. 19
      weed/cluster/cluster.go
  27. 2
      weed/command/benchmark.go
  28. 31
      weed/command/filer.go
  29. 3
      weed/command/filer_sync.go
  30. 2
      weed/command/iam.go
  31. 42
      weed/command/master.go
  32. 2
      weed/command/master_follower.go
  33. 2
      weed/command/mount.go
  34. 24
      weed/command/mount_std.go
  35. 3
      weed/command/s3.go
  36. 3
      weed/command/server.go
  37. 21
      weed/filer/filechunks.go
  38. 10
      weed/filer/filechunks2_test.go
  39. 17
      weed/filer/filechunks_read.go
  40. 2
      weed/filer/filer.go
  41. 3
      weed/filer/meta_aggregator.go
  42. 6
      weed/filer/redis/universal_redis_store.go
  43. 13
      weed/filer/stream.go
  44. 32
      weed/iamapi/iamapi_management_handlers.go
  45. 5
      weed/iamapi/iamapi_response.go
  46. 2
      weed/iamapi/iamapi_server.go
  47. 21
      weed/iamapi/iamapi_test.go
  48. 6
      weed/mount/filehandle.go
  49. 1
      weed/mount/filehandle_map.go
  50. 7
      weed/mount/page_writer/page_chunk_swapfile.go
  51. 3
      weed/mount/page_writer/upload_pipeline.go
  52. 5
      weed/mount/weedfs.go
  53. 9
      weed/mount/weedfs_attr.go
  54. 1
      weed/mount/weedfs_filehandle.go
  55. 17
      weed/mount/weedfs_grpc_server.go
  56. 9
      weed/mount/weedfs_quota.go
  57. 1
      weed/pb/Makefile
  58. 13
      weed/pb/filer.proto
  59. 920
      weed/pb/filer_pb/filer.pb.go
  60. 37
      weed/pb/filer_pb/filer_grpc.pb.go
  61. 11
      weed/pb/grpc_client_server.go
  62. 4
      weed/pb/iam_pb/iam.pb.go
  63. 1
      weed/pb/iam_pb/iam_grpc.pb.go
  64. 49
      weed/pb/master.proto
  65. 1606
      weed/pb/master_pb/master.pb.go
  66. 145
      weed/pb/master_pb/master_grpc.pb.go
  67. 4
      weed/pb/messaging_pb/messaging.pb.go
  68. 1
      weed/pb/messaging_pb/messaging_grpc.pb.go
  69. 25
      weed/pb/mount.proto
  70. 208
      weed/pb/mount_pb/mount.pb.go
  71. 100
      weed/pb/mount_pb/mount_grpc.pb.go
  72. 4
      weed/pb/remote_pb/remote.pb.go
  73. 15
      weed/pb/server_address.go
  74. 14
      weed/pb/volume_server.proto
  75. 881
      weed/pb/volume_server_pb/volume_server.pb.go
  76. 37
      weed/pb/volume_server_pb/volume_server_grpc.pb.go
  77. 28
      weed/remote_storage/remote_storage.go
  78. 27
      weed/s3api/filer_multipart.go
  79. 6
      weed/s3api/filer_multipart_test.go
  80. 20
      weed/s3api/s3api_bucket_handlers.go
  81. 17
      weed/s3api/s3api_object_handlers.go
  82. 44
      weed/s3api/s3api_object_multipart_handlers.go
  83. 1
      weed/s3api/s3api_server.go
  84. 6
      weed/s3api/s3err/s3api_errors.go
  85. 126
      weed/server/filer_grpc_server.go
  86. 177
      weed/server/filer_grpc_server_admin.go
  87. 2
      weed/server/filer_server.go
  88. 6
      weed/server/filer_server_handlers_read_dir.go
  89. 2
      weed/server/filer_server_handlers_tagging.go
  90. 12
      weed/server/filer_server_handlers_write_autochunk.go
  91. 11
      weed/server/filer_server_handlers_write_merge.go
  92. 8
      weed/server/filer_server_handlers_write_upload.go
  93. 6
      weed/server/filer_ui/breadcrumb.go
  94. 247
      weed/server/filer_ui/filer.html
  95. 15
      weed/server/master_grpc_server.go
  96. 42
      weed/server/master_grpc_server_admin.go
  97. 66
      weed/server/master_grpc_server_raft.go
  98. 2
      weed/server/master_grpc_server_volume.go
  99. 114
      weed/server/master_server.go
  100. 2
      weed/server/master_server_handlers_admin.go

9
.github/workflows/binaries_dev.yml

@ -4,9 +4,14 @@ on:
push:
branches: [ master ]
permissions:
contents: read
jobs:
cleanup:
permissions:
contents: write # for mknejp/delete-release-assets to delete release assets
runs-on: ubuntu-latest
steps:
@ -21,6 +26,8 @@ jobs:
weed-*
build_dev_linux_windows:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
needs: cleanup
runs-on: ubuntu-latest
strategy:
@ -68,6 +75,8 @@ jobs:
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
build_dev_darwin:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
needs: build_dev_linux_windows
runs-on: ubuntu-latest
strategy:

5
.github/workflows/binaries_release0.yml

@ -11,9 +11,14 @@ on:
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_windows:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:

5
.github/workflows/binaries_release1.yml

@ -11,9 +11,14 @@ on:
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_linux:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:

5
.github/workflows/binaries_release2.yml

@ -11,9 +11,14 @@ on:
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_darwin:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:

5
.github/workflows/binaries_release3.yml

@ -11,9 +11,14 @@ on:
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
permissions:
contents: read
jobs:
build-release-binaries_freebsd:
permissions:
contents: write # for wangyoucao577/go-release-action to upload release assets
runs-on: ubuntu-latest
strategy:
matrix:

2
.github/workflows/container_dev.yml

@ -20,7 +20,7 @@ jobs:
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
with:
images: |
chrislusf/seaweedfs

2
.github/workflows/container_latest.yml

@ -21,7 +21,7 @@ jobs:
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
with:
images: |
chrislusf/seaweedfs

2
.github/workflows/container_release1.yml

@ -20,7 +20,7 @@ jobs:
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
with:
images: |
chrislusf/seaweedfs

2
.github/workflows/container_release2.yml

@ -21,7 +21,7 @@ jobs:
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
with:
images: |
chrislusf/seaweedfs

2
.github/workflows/container_release3.yml

@ -21,7 +21,7 @@ jobs:
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
with:
images: |
chrislusf/seaweedfs

4
README.md

@ -54,6 +54,7 @@ Table of Contents
* [Quick Start](#quick-start)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@ -82,6 +83,9 @@ Table of Contents
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
## Quick Start SeaweedFS S3 on AWS ##
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
## Introduction ##
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:

2
docker/Dockerfile.go_build

@ -1,4 +1,4 @@
FROM golang:1.17-alpine as builder
FROM golang:1.18-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

2
docker/Dockerfile.go_build_large

@ -1,4 +1,4 @@
FROM golang:1.17-alpine as builder
FROM golang:1.18-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

4
docker/Dockerfile.rocksdb_large

@ -1,9 +1,9 @@
FROM golang:1.17-buster as builder
FROM golang:1.18-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ENV ROCKSDB_VERSION v6.22.1
ENV ROCKSDB_VERSION v7.0.4
# build RocksDB
RUN cd /tmp && \

3
docker/Makefile

@ -55,6 +55,9 @@ cluster: build
2clusters: build
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
hashicorp_raft: build
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
s3tests: build s3tests_build
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up

89
docker/compose/local-hashicorp-raft-compose.yml

@ -0,0 +1,89 @@
version: '2'
services:
master0:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
volumes:
- ./master/0:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master1:
image: chrislusf/seaweedfs:local
ports:
- 9334:9334
- 19334:19334
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
volumes:
- ./master/1:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
master2:
image: chrislusf/seaweedfs:local
ports:
- 9335:9335
- 19335:19335
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
volumes:
- ./master/2:/data
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volume1:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
depends_on:
- master0
- master1
volume2:
image: chrislusf/seaweedfs:local
ports:
- 8082:8082
- 18082:18082
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
depends_on:
- master0
- master1
volume3:
image: chrislusf/seaweedfs:local
ports:
- 8083:8083
- 18083:18083
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
depends_on:
- master0
- master1
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8111:8111
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
depends_on:
- master0
- master1
- volume1
- volume2
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333
command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"'
depends_on:
- master0
- master1
- volume1
- volume2
- filer

4
docker/compose/local-s3tests-compose.yml

@ -24,7 +24,7 @@ services:
- 8888:8888
- 18888:18888
- 8000:8000
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
@ -38,7 +38,7 @@ services:
S3TEST_CONF: "s3tests.conf"
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)"
depends_on:
- master
- volume

153
go.mod

@ -4,13 +4,13 @@ go 1.18
require (
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/pubsub v1.19.0
cloud.google.com/go/storage v1.21.0
cloud.google.com/go/pubsub v1.20.0
cloud.google.com/go/storage v1.22.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.14.0
github.com/OneOfOne/xxhash v1.2.2
github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.43.25
github.com/OneOfOne/xxhash v1.2.8
github.com/Shopify/sarama v1.32.0
github.com/aws/aws-sdk-go v1.43.41
github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
@ -33,11 +33,10 @@ require (
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.17.0
github.com/frankban/quicktest v1.7.2 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.4.1
github.com/go-redsync/redsync/v4 v4.5.0
github.com/go-sql-driver/mysql v1.6.0
github.com/go-stack/stack v1.8.0 // indirect
github.com/go-zookeeper/zk v1.0.2 // indirect
@ -50,9 +49,9 @@ require (
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/uuid v1.3.0
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/gorilla/mux v1.7.4
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
github.com/googleapis/gax-go/v2 v2.3.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.0 // indirect
@ -65,41 +64,40 @@ require (
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/reedsolomon v1.9.2
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/reedsolomon v1.9.16
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.4
github.com/linxGnu/grocksdb v1.6.38
github.com/magiconair/properties v1.8.1 // indirect
github.com/mailru/easyjson v0.7.1 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/lib/pq v1.10.5
github.com/linxGnu/grocksdb v1.7.0
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.3 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/olivere/elastic/v7 v7.0.19
github.com/pelletier/go-toml v1.7.0 // indirect
github.com/olivere/elastic/v7 v7.0.32
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/peterh/liner v1.2.2
github.com/pierrec/lz4 v2.2.7+incompatible // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.7.0 // indirect
github.com/spf13/cast v1.3.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.4.0
github.com/spf13/viper v1.11.0
github.com/streadway/amqp v1.0.0
github.com/stretchr/testify v1.7.1
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
@ -114,88 +112,103 @@ require (
github.com/viant/ptrie v0.3.0
github.com/viant/toolbox v0.33.2 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.0.2 // indirect
github.com/xdg-go/scram v1.1.0 // indirect
github.com/xdg-go/stringprep v1.0.2 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.0
go.mongodb.org/mongo-driver v1.8.4
go.etcd.io/etcd/client/v3 v3.5.3
go.mongodb.org/mongo-driver v1.9.0
go.opencensus.io v0.23.0 // indirect
gocloud.dev v0.24.0
gocloud.dev/pubsub/natspubsub v0.20.0
gocloud.dev/pubsub/rabbitpubsub v0.24.0
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
gocloud.dev v0.25.0
gocloud.dev/pubsub/natspubsub v0.25.0
gocloud.dev/pubsub/rabbitpubsub v0.25.0
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
golang.org/x/net v0.0.0-20220412020605-290c469a71a5
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.73.0
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/api v0.74.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
google.golang.org/grpc v1.45.0
google.golang.org/protobuf v1.28.0
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.35.24 // indirect
modernc.org/ccgo/v3 v3.15.17 // indirect
modernc.org/ccgo/v3 v3.15.18 // indirect
modernc.org/libc v1.14.12 // indirect
modernc.org/mathutil v1.4.1 // indirect
modernc.org/memory v1.0.7 // indirect
modernc.org/opt v0.1.1 // indirect
modernc.org/sqlite v1.15.3
modernc.org/sqlite v1.16.0
modernc.org/strutil v1.1.1 // indirect
modernc.org/token v1.0.0 // indirect
)
require (
github.com/fluent/fluent-logger-golang v1.8.0
github.com/Jille/raft-grpc-transport v1.2.0
github.com/fluent/fluent-logger-golang v1.9.0
github.com/hanwen/go-fuse/v2 v2.1.0
github.com/hashicorp/raft v1.3.7
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
)
require (
cloud.google.com/go/compute v1.5.0 // indirect
cloud.google.com/go/iam v0.1.1 // indirect
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
github.com/aws/smithy-go v1.8.0 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fclairamb/go-log v0.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 // indirect
github.com/googleapis/go-type-adapters v1.0.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/nats-io/nats.go v1.11.0 // indirect
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.3 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.3 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.0 // indirect
golang.org/x/mod v0.5.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
lukechampine.com/uint128 v1.1.1 // indirect

730
go.sum
File diff suppressed because it is too large
View File

4
k8s/helm_charts2/Chart.yaml

@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "2.95"
version: "2.95"
appVersion: "2.99"
version: "2.99"

32
k8s/helm_charts2/templates/filer-statefulset.yaml

@ -154,13 +154,16 @@ spec:
{{- end }}
{{- end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
{{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}
volumeMounts:
- name: seaweedfs-filer-log-volume
mountPath: "/logs/"
- mountPath: /etc/sw
name: config-users
readOnly: true
{{- if .Values.filer.enablePVC }}
- name: data-filer
mountPath: /data
{{- end }}
{{- if .Values.global.enableSecurity }}
- name: security-config
readOnly: true
@ -183,7 +186,6 @@ spec:
mountPath: /usr/local/share/ca-certificates/client/
{{- end }}
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
{{- end }}
ports:
- containerPort: {{ .Values.filer.port }}
name: swfs-filer
@ -250,16 +252,18 @@ spec:
nodeSelector:
{{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
{{- end }}
{{/* volumeClaimTemplates:*/}}
{{/* - metadata:*/}}
{{/* name: data-{{ .Release.Namespace }}*/}}
{{/* spec:*/}}
{{/* accessModes:*/}}
{{/* - ReadWriteOnce*/}}
{{/* resources:*/}}
{{/* requests:*/}}
{{/* storage: {{ .Values.filer.storage }}*/}}
{{/* {{- if .Values.filer.storageClass }}*/}}
{{/* storageClassName: {{ .Values.filer.storageClass }}*/}}
{{/* {{- end }}*/}}
{{- if .Values.filer.enablePVC }}
volumeClaimTemplates:
- metadata:
name: data-filer
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.filer.storage }}
{{- if .Values.filer.storageClass }}
storageClassName: {{ .Values.filer.storageClass }}
{{- end }}
{{- end }}
{{- end }}

6
k8s/helm_charts2/values.yaml

@ -41,8 +41,7 @@ master:
grpcPort: 19333
ipBind: "0.0.0.0"
volumePreallocate: false
#Master stops directing writes to oversized volumes
volumeSizeLimitMB: 30000
volumeSizeLimitMB: 1000
loggingOverrideLevel: null
#number of seconds between heartbeats, default 5
pulseSeconds: null
@ -62,6 +61,8 @@ master:
extraVolumes: ""
extraVolumeMounts: ""
# enablePVC will create a pvc for filer for data persistence.
enablePVC: false
# storage and storageClass are the settings for configuring stateful
# storage for the master pods. storage should be set to the disk size of
# the attached volume. storageClass is the class of storage which defaults
@ -358,6 +359,7 @@ filer:
WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600"
# enable usage of memsql as filer backend
WEED_MYSQL_INTERPOLATEPARAMS: "true"
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
WEED_LEVELDB2_ENABLED: "false"
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"

13
other/java/client/src/main/proto/filer.proto

@ -48,6 +48,9 @@ service SeaweedFiler {
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
}
rpc Ping (PingRequest) returns (PingResponse) {
}
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
}
@ -311,6 +314,16 @@ message StatisticsResponse {
uint64 file_count = 6;
}
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
}
message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {

2
other/java/hdfs-over-ftp/pom.xml

@ -36,7 +36,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.2</version>
<version>3.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>

2
other/java/hdfs3/pom.xml

@ -6,7 +6,7 @@
<properties>
<seaweedfs.client.version>2.85</seaweedfs.client.version>
<hadoop.version>3.1.4</hadoop.version>
<hadoop.version>3.2.3</hadoop.version>
</properties>
<groupId>com.github.chrislusf</groupId>

19
weed/cluster/cluster.go

@ -10,6 +10,7 @@ import (
const (
MasterType = "master"
VolumeServerType = "volumeServer"
FilerType = "filer"
BrokerType = "broker"
)
@ -80,6 +81,15 @@ func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress
},
}
case MasterType:
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
}
return nil
}
@ -119,6 +129,15 @@ func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddr
}
}
case MasterType:
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
}
return nil
}

2
weed/command/benchmark.go

@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile()
}
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected()

31
weed/command/filer.go

@ -5,6 +5,7 @@ import (
"net"
"net/http"
"os"
"runtime"
"time"
"google.golang.org/grpc/reflection"
@ -29,7 +30,7 @@ var (
)
type FilerOptions struct {
masters []pb.ServerAddress
masters map[string]pb.ServerAddress
mastersString *string
ip *string
bindIp *string
@ -89,6 +90,7 @@ func init() {
filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
// start webdav on filer
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@ -171,7 +173,7 @@ func runFiler(cmd *Command, args []string) bool {
}()
}
f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
f.startFiler()
@ -247,18 +249,6 @@ func (fo *FilerOptions) startFiler() {
glog.Fatalf("Filer listener error: %v", e)
}
// start on local unix socket
if *fo.localSocket == "" {
*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
}
}
filerSocketListener, err := net.Listen("unix", *fo.localSocket)
if err != nil {
glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
}
// starting grpc server
grpcPort := *fo.portGrpc
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
@ -274,9 +264,22 @@ func (fo *FilerOptions) startFiler() {
go grpcS.Serve(grpcL)
httpS := &http.Server{Handler: defaultMux}
if runtime.GOOS != "windows" {
if *fo.localSocket == "" {
*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
}
}
go func() {
// start on local unix socket
filerSocketListener, err := net.Listen("unix", *fo.localSocket)
if err != nil {
glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
}
httpS.Serve(filerSocketListener)
}()
}
if filerLocalListener != nil {
go func() {
if err := httpS.Serve(filerLocalListener); err != nil {

3
weed/command/filer_sync.go

@ -267,8 +267,11 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
return nil
}
key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
if !dataSink.IsIncremental() {
return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
}
return nil
}
// handle new entries
if filer_pb.IsCreate(resp) {

2
weed/command/iam.go

@ -67,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
}
}
masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
router := mux.NewRouter().SkipClean(true)
_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
Masters: masters,

42
weed/command/master.go

@ -1,9 +1,9 @@
package command
import (
"golang.org/x/exp/slices"
"net/http"
"os"
"sort"
"strings"
"time"
@ -48,6 +48,8 @@ type MasterOptions struct {
metricsHttpPort *int
heartbeatInterval *time.Duration
electionTimeout *time.Duration
raftHashicorp *bool
raftBootstrap *bool
}
func init() {
@ -71,6 +73,8 @@ func init() {
m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
}
var cmdMaster = &Command{
@ -132,8 +136,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
masterPeers := make(map[string]pb.ServerAddress)
for _, peer := range peers {
masterPeers[string(peer)] = peer
}
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
@ -144,20 +153,32 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
// start raftServer
raftServerOption := &weed_server.RaftServerOption{
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
Peers: peers,
Peers: masterPeers,
ServerAddr: myMasterAddress,
DataDir: util.ResolvePath(*masterOption.metaFolder),
Topo: ms.Topo,
RaftResumeState: *masterOption.raftResumeState,
HeartbeatInterval: *masterOption.heartbeatInterval,
ElectionTimeout: *masterOption.electionTimeout,
RaftBootstrap: *m.raftBootstrap,
}
var raftServer *weed_server.RaftServer
var err error
if *m.raftHashicorp {
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
glog.Fatalf("NewHashicorpRaftServer: %s", err)
}
raftServer, err := weed_server.NewRaftServer(raftServerOption)
} else {
raftServer, err = weed_server.NewRaftServer(raftServerOption)
if raftServer == nil {
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
}
}
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
if *m.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
}
// starting grpc server
grpcPort := *masterOption.portGrpc
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
@ -166,7 +187,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
if *m.raftHashicorp {
raftServer.TransportManager.Register(grpcS)
} else {
protobuf.RegisterRaftServer(grpcS, raftServer)
}
reflection.Register(grpcS)
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
if grpcLocalL != nil {
@ -174,14 +199,17 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
}
go grpcS.Serve(grpcL)
timeSleep := 1500 * time.Millisecond
if !*m.raftHashicorp {
go func() {
time.Sleep(1500 * time.Millisecond)
time.Sleep(timeSleep)
if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
raftServer.DoJoinCommand()
}
}
}()
}
go ms.MasterClient.KeepConnectedToMaster()
@ -246,8 +274,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
}
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
sort.Slice(peers, func(i, j int) bool {
return strings.Compare(string(peers[i]), string(peers[j])) < 0
slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
return strings.Compare(string(a), string(b)) < 0
})
if len(peers) <= 0 {
return true

2
weed/command/master_follower.go

@ -83,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
func startMasterFollower(masterOptions MasterOptions) {
// collect settings from main masters
masters := pb.ServerAddresses(*mf.peers).ToAddresses()
masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
var err error
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")

2
weed/command/mount.go

@ -29,6 +29,7 @@ type MountOptions struct {
readOnly *bool
debug *bool
debugPort *int
localSocket *string
}
var (
@ -63,6 +64,7 @@ func init() {
mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

24
weed/command/mount_std.go

@ -12,9 +12,12 @@ import (
"github.com/chrislusf/seaweedfs/weed/mount/unmount"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/hanwen/go-fuse/v2/fuse"
"google.golang.org/grpc/reflection"
"net"
"net/http"
"os"
"os/user"
@ -98,6 +101,22 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
unmount.Unmount(dir)
// start on local unix socket
if *option.localSocket == "" {
mountDirHash := util.HashToInt32([]byte(dir))
if mountDirHash < 0 {
mountDirHash = -mountDirHash
}
*option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash)
}
if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
}
montSocketListener, err := net.Listen("unix", *option.localSocket)
if err != nil {
glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
}
// detect mount folder mode
if *option.dirAutoCreate {
os.MkdirAll(dir, os.FileMode(0777)&^umask)
@ -229,6 +248,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
unmount.Unmount(dir)
})
grpcS := pb.NewGrpcServer()
mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
reflection.Register(grpcS)
go grpcS.Serve(montSocketListener)
seaweedFileSystem.StartBackgroundTasks()
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)

3
weed/command/s3.go

@ -33,6 +33,7 @@ type S3Options struct {
tlsCertificate *string
metricsHttpPort *int
allowEmptyFolder *bool
allowDeleteBucketNotEmpty *bool
auditLogConfig *string
localFilerSocket *string
}
@ -49,6 +50,7 @@ func init() {
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
}
var cmdS3 = &Command{
@ -185,6 +187,7 @@ func (s3opt *S3Options) startS3Server() bool {
BucketsPath: filerBucketsPath,
GrpcDialOption: grpcDialOption,
AllowEmptyFolder: *s3opt.allowEmptyFolder,
AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
LocalFilerSocket: s3opt.localFilerSocket,
})
if s3ApiServer_err != nil {

3
weed/command/server.go

@ -138,6 +138,7 @@ func init() {
s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
@ -191,7 +192,7 @@ func runServer(cmd *Command, args []string) bool {
// ip address
masterOptions.ip = serverIp
masterOptions.ipBind = serverBindIp
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
filerOptions.ip = serverIp
filerOptions.bindIp = serverBindIp
s3Options.bindIp = serverBindIp

21
weed/filer/filechunks.go

@ -4,8 +4,8 @@ import (
"bytes"
"fmt"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"golang.org/x/exp/slices"
"math"
"sort"
"sync"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@ -23,6 +23,9 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
}
func FileSize(entry *filer_pb.Entry) (size uint64) {
if entry == nil || entry.Attributes == nil {
return 0
}
fileSize := entry.Attributes.FileSize
if entry.RemoteEntry != nil {
if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
@ -251,19 +254,17 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
if true {
return visibles2, err
}
sort.Slice(chunks, func(i, j int) bool {
if chunks[i].Mtime == chunks[j].Mtime {
filer_pb.EnsureFid(chunks[i])
filer_pb.EnsureFid(chunks[j])
if chunks[i].Fid == nil || chunks[j].Fid == nil {
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
if a.Mtime == b.Mtime {
filer_pb.EnsureFid(a)
filer_pb.EnsureFid(b)
if a.Fid == nil || b.Fid == nil {
return true
}
return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
return a.Fid.FileKey < b.Fid.FileKey
}
return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run
return a.Mtime < b.Mtime
})
for _, chunk := range chunks {
// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))

10
weed/filer/filechunks2_test.go

@ -1,7 +1,7 @@
package filer
import (
"sort"
"golang.org/x/exp/slices"
"testing"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -34,11 +34,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
}
func printChunks(name string, chunks []*filer_pb.FileChunk) {
sort.Slice(chunks, func(i, j int) bool {
if chunks[i].Offset == chunks[j].Offset {
return chunks[i].Mtime < chunks[j].Mtime
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
if a.Offset == b.Offset {
return a.Mtime < b.Mtime
}
return chunks[i].Offset < chunks[j].Offset
return a.Offset < b.Offset
})
for _, chunk := range chunks {
glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))

17
weed/filer/filechunks_read.go

@ -2,7 +2,7 @@ package filer
import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"sort"
"golang.org/x/exp/slices"
)
func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
@ -22,17 +22,14 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
isStart: false,
})
}
sort.Slice(points, func(i, j int) bool {
if points[i].x != points[j].x {
return points[i].x < points[j].x
slices.SortFunc(points, func(a, b *Point) bool {
if a.x != b.x {
return a.x < b.x
}
if points[i].ts != points[j].ts {
return points[i].ts < points[j].ts
if a.ts != b.ts {
return a.ts < b.ts
}
if !points[i].isStart {
return true
}
return false
return !a.isStart
})
var prevX int64

2
weed/filer/filer.go

@ -49,7 +49,7 @@ type Filer struct {
UniqueFileId uint32
}
func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption,
filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
f := &Filer{
MasterClient: wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),

3
weed/filer/meta_aggregator.go

@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
}
} else {
if _, found := ma.peerStatues[address]; found {
ma.peerStatues[address] -= 1
}
if ma.peerStatues[address] <= 0 {
delete(ma.peerStatues, address)
}
}

6
weed/filer/redis/universal_redis_store.go

@ -3,7 +3,7 @@ package redis
import (
"context"
"fmt"
"sort"
"golang.org/x/exp/slices"
"strings"
"time"
@ -157,8 +157,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
}
// sort
sort.Slice(members, func(i, j int) bool {
return strings.Compare(members[i], members[j]) < 0
slices.SortFunc(members, func(a, b string) bool {
return strings.Compare(a, b) < 0
})
// limit

13
weed/filer/stream.go

@ -3,6 +3,7 @@ package filer
import (
"bytes"
"fmt"
"golang.org/x/exp/slices"
"io"
"math"
"sort"
@ -39,11 +40,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool {
if len(a) != len(b) {
return false
}
sort.Slice(a, func(i, j int) bool {
return strings.Compare(a[i].ETag, a[j].ETag) < 0
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
})
sort.Slice(b, func(i, j int) bool {
return strings.Compare(b[i].ETag, b[j].ETag) < 0
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
})
for i := 0; i < len(a); i++ {
if a[i].ETag != b[i].ETag {
@ -179,8 +180,8 @@ var _ = io.ReaderAt(&ChunkStreamReader{})
func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
sort.Slice(chunkViews, func(i, j int) bool {
return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset
slices.SortFunc(chunkViews, func(a, b *ChunkView) bool {
return a.LogicOffset < b.LogicOffset
})
var totalSize int64

32
weed/iamapi/iamapi_management_handlers.go

@ -4,10 +4,6 @@ import (
"crypto/sha1"
"encoding/json"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"math/rand"
"net/http"
"net/url"
@ -16,6 +12,11 @@ import (
"sync"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/aws/aws-sdk-go/service/iam"
)
@ -155,6 +156,22 @@ func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName str
return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
}
func (iama *IamApiServer) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp UpdateUserResponse, err error) {
userName := values.Get("UserName")
newUserName := values.Get("NewUserName")
if newUserName != "" {
for _, ident := range s3cfg.Identities {
if userName == ident.Name {
ident.Name = newUserName
return resp, nil
}
}
} else {
return resp, nil
}
return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
}
func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) {
if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil {
return PolicyDocument{}, err
@ -396,6 +413,13 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
return
}
changed = false
case "UpdateUser":
response, err = iama.UpdateUser(s3cfg, values)
if err != nil {
glog.Errorf("UpdateUser: %+v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
case "DeleteUser":
userName := values.Get("UserName")
response, err = iama.DeleteUser(s3cfg, userName)

5
weed/iamapi/iamapi_response.go

@ -66,6 +66,11 @@ type GetUserResponse struct {
} `xml:"GetUserResult"`
}
type UpdateUserResponse struct {
CommonResponse
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateUserResponse"`
}
type CreateAccessKeyResponse struct {
CommonResponse
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"`

2
weed/iamapi/iamapi_server.go

@ -33,7 +33,7 @@ type IamS3ApiConfigure struct {
}
type IamServerOption struct {
Masters []pb.ServerAddress
Masters map[string]pb.ServerAddress
Filer pb.ServerAddress
Port int
GrpcDialOption grpc.DialOption

21
weed/iamapi/iamapi_test.go

@ -2,6 +2,10 @@ package iamapi
import (
"encoding/xml"
"net/http"
"net/http/httptest"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/iam"
@ -9,9 +13,6 @@ import (
"github.com/gorilla/mux"
"github.com/jinzhu/copier"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
@ -161,8 +162,20 @@ func TestGetUserPolicy(t *testing.T) {
assert.Equal(t, http.StatusOK, response.Code)
}
func TestDeleteUser(t *testing.T) {
func TestUpdateUser(t *testing.T) {
userName := aws.String("Test")
newUserName := aws.String("Test-New")
params := &iam.UpdateUserInput{NewUserName: newUserName, UserName: userName}
req, _ := iam.New(session.New()).UpdateUserRequest(params)
_ = req.Build()
out := UpdateUserResponse{}
response, err := executeRequest(req.HTTPRequest, out)
assert.Equal(t, nil, err)
assert.Equal(t, http.StatusOK, response.Code)
}
func TestDeleteUser(t *testing.T) {
userName := aws.String("Test-New")
params := &iam.DeleteUserInput{UserName: userName}
req, _ := iam.New(session.New()).DeleteUserRequest(params)
_ = req.Build()

6
weed/mount/filehandle.go

@ -5,8 +5,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"golang.org/x/exp/slices"
"io"
"sort"
"sync"
)
@ -76,8 +76,8 @@ func (fh *FileHandle) addChunks(chunks []*filer_pb.FileChunk) {
}
// sort incoming chunks
sort.Slice(chunks, func(i, j int) bool {
return lessThan(chunks[i], chunks[j])
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
return lessThan(a, b)
})
glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks))

1
weed/mount/filehandle_map.go

@ -49,6 +49,7 @@ func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *fil
} else {
fh.counter++
}
fh.entry = entry
return fh
}

7
weed/mount/page_writer/page_chunk_swapfile.go

@ -5,6 +5,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/mem"
"os"
"sync"
)
var (
@ -17,6 +18,7 @@ type SwapFile struct {
dir string
file *os.File
logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
logicToActualChunkIndexLock sync.Mutex
chunkSize int64
freeActualChunkList []ActualChunkIndex
}
@ -52,6 +54,8 @@ func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapF
return nil
}
}
sf.logicToActualChunkIndexLock.Lock()
defer sf.logicToActualChunkIndexLock.Unlock()
actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
if !found {
if len(sf.freeActualChunkList) > 0 {
@ -72,6 +76,9 @@ func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapF
}
func (sc *SwapFileChunk) FreeResource() {
sc.swapfile.logicToActualChunkIndexLock.Lock()
defer sc.swapfile.logicToActualChunkIndexLock.Unlock()
sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex)
}

3
weed/mount/page_writer/upload_pipeline.go

@ -187,6 +187,9 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
func (up *UploadPipeline) Shutdown() {
up.swapFile.FreeResource()
up.sealedChunksLock.Lock()
defer up.sealedChunksLock.Unlock()
for logicChunkIndex, sealedChunk := range up.sealedChunks {
sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
}

5
weed/mount/weedfs.go

@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
@ -59,6 +60,7 @@ type WFS struct {
// https://dl.acm.org/doi/fullHtml/10.1145/3310148
// follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go
fuse.RawFileSystem
mount_pb.UnimplementedSeaweedMountServer
fs.Inode
option *Option
metaCache *meta_cache.MetaCache
@ -129,6 +131,9 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
}
var found bool
if fh, found = wfs.fhmap.FindFileHandle(inode); found {
if fh.entry.Attributes == nil {
fh.entry.Attributes = &filer_pb.FuseAttributes{}
}
return path, fh, fh.entry, fuse.OK
}
entry, status = wfs.maybeLoadEntry(path)

9
weed/mount/weedfs_attr.go

@ -98,15 +98,14 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
}
}
if mtime, ok := input.GetMTime(); ok {
entry.Attributes.Mtime = mtime.Unix()
}
if atime, ok := input.GetATime(); ok {
entry.Attributes.Mtime = atime.Unix()
}
entry.Attributes.Mtime = time.Now().Unix()
if mtime, ok := input.GetMTime(); ok {
entry.Attributes.Mtime = mtime.Unix()
}
out.AttrValid = 1
wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

1
weed/mount/weedfs_filehandle.go

@ -10,7 +10,6 @@ func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHa
_, _, entry, status = wfs.maybeReadEntry(inode)
if status == fuse.OK {
fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry)
fileHandle.entry = entry
}
return
}

17
weed/mount/weedfs_grpc_server.go

@ -0,0 +1,17 @@
package mount
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
)
func (wfs *WFS) Configure(ctx context.Context, request *mount_pb.ConfigureRequest) (*mount_pb.ConfigureResponse, error) {
if wfs.option.Collection == "" {
return nil, fmt.Errorf("mount quota only works when mounted to a new folder with a collection")
}
glog.V(0).Infof("quota changed from %d to %d", wfs.option.Quota, request.CollectionCapacity)
wfs.option.Quota = request.GetCollectionCapacity()
return &mount_pb.ConfigureResponse{}, nil
}

9
weed/mount/weedfs_quota.go

@ -10,12 +10,14 @@ import (
func (wfs *WFS) loopCheckQuota() {
for {
time.Sleep(61 * time.Second)
if wfs.option.Quota <= 0 {
return
continue
}
for {
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.StatisticsRequest{
@ -47,7 +49,6 @@ func (wfs *WFS) loopCheckQuota() {
glog.Warningf("read quota usage: %v", err)
}
time.Sleep(61 * time.Second)
}
}

1
weed/pb/Makefile

@ -8,6 +8,7 @@ gen:
protoc filer.proto --go_out=./filer_pb --go-grpc_out=./filer_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc remote.proto --go_out=./remote_pb --go-grpc_out=./remote_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc iam.proto --go_out=./iam_pb --go-grpc_out=./iam_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc mount.proto --go_out=./mount_pb --go-grpc_out=./mount_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc messaging.proto --go_out=./messaging_pb --go-grpc_out=./messaging_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
# protoc filer.proto --java_out=../../other/java/client/src/main/java
cp filer.proto ../../other/java/client/src/main/proto

13
weed/pb/filer.proto

@ -48,6 +48,9 @@ service SeaweedFiler {
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
}
rpc Ping (PingRequest) returns (PingResponse) {
}
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
}
@ -311,6 +314,16 @@ message StatisticsResponse {
uint64 file_count = 6;
}
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
}
message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {

920
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File

37
weed/pb/filer_pb/filer_grpc.pb.go

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedFilerClient is the client API for SeaweedFiler service.
@ -31,6 +30,7 @@ type SeaweedFilerClient interface {
CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error)
DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error)
SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error)
SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error)
@ -212,6 +212,15 @@ func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsReque
return out, nil
}
func (c *seaweedFilerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
out := new(PingResponse)
err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) {
out := new(GetFilerConfigurationResponse)
err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, opts...)
@ -369,6 +378,7 @@ type SeaweedFilerServer interface {
CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error)
DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
Ping(context.Context, *PingRequest) (*PingResponse, error)
GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error)
SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error
SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error
@ -423,6 +433,9 @@ func (UnimplementedSeaweedFilerServer) DeleteCollection(context.Context, *Delete
func (UnimplementedSeaweedFilerServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented")
}
func (UnimplementedSeaweedFilerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedSeaweedFilerServer) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetFilerConfiguration not implemented")
}
@ -700,6 +713,24 @@ func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedFilerServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/filer_pb.SeaweedFiler/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).Ping(ctx, req.(*PingRequest))
}
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetFilerConfigurationRequest)
if err := dec(in); err != nil {
@ -909,6 +940,10 @@ var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{
MethodName: "Statistics",
Handler: _SeaweedFiler_Statistics_Handler,
},
{
MethodName: "Ping",
Handler: _SeaweedFiler_Ping_Handler,
},
{
MethodName: "GetFilerConfiguration",
Handler: _SeaweedFiler_GetFilerConfiguration_Handler,

11
weed/pb/grpc_client_server.go

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"math/rand"
"net/http"
@ -206,7 +207,15 @@ func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption g
}
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
func WithVolumeServerClient(streamingMode bool, volumeServer ServerAddress, grpcDialOption grpc.DialOption, fn func(client volume_server_pb.VolumeServerClient) error) error {
return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
client := volume_server_pb.NewVolumeServerClient(grpcConnection)
return fn(client)
}, volumeServer.ToGrpcAddress(), grpcDialOption)
}
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
for _, masterGrpcAddress := range masterGrpcAddresses {
err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {

4
weed/pb/iam_pb/iam.pb.go

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.19.4
// source: iam.proto
package iam_pb

1
weed/pb/iam_pb/iam_grpc.pb.go

@ -8,7 +8,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.

49
weed/pb/master.proto

@ -35,7 +35,14 @@ service Seaweed {
}
rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
}
rpc Ping (PingRequest) returns (PingResponse) {
}
rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) {
}
rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) {
}
rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) {
}
}
//////////////////////////////////////////////////
@ -140,6 +147,8 @@ message VolumeLocation {
string leader = 5; // optional when leader is not itself
string data_center = 6; // optional when DataCenter is in use
uint32 grpc_port = 7;
repeated uint32 new_ec_vids = 8;
repeated uint32 deleted_ec_vids = 9;
}
message ClusterNodeUpdate {
@ -281,6 +290,8 @@ message LookupEcVolumeResponse {
message VacuumVolumeRequest {
float garbage_threshold = 1;
uint32 volume_id = 2;
string collection = 3;
}
message VacuumVolumeResponse {
}
@ -328,3 +339,39 @@ message ReleaseAdminTokenRequest {
}
message ReleaseAdminTokenResponse {
}
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
}
message RaftAddServerRequest {
string id = 1;
string address = 2;
bool voter = 3;
}
message RaftAddServerResponse {
}
message RaftRemoveServerRequest {
string id = 1;
bool force = 2;
}
message RaftRemoveServerResponse {
}
message RaftListClusterServersRequest {
}
message RaftListClusterServersResponse {
message ClusterServers {
string id = 1;
string address = 2;
string suffrage = 3; //
}
repeated ClusterServers cluster_servers = 1;
}

1606
weed/pb/master_pb/master.pb.go
File diff suppressed because it is too large
View File

145
weed/pb/master_pb/master_grpc.pb.go

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedClient is the client API for Seaweed service.
@ -32,6 +31,10 @@ type SeaweedClient interface {
ListClusterNodes(ctx context.Context, in *ListClusterNodesRequest, opts ...grpc.CallOption) (*ListClusterNodesResponse, error)
LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error)
ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error)
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error)
RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error)
RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error)
}
type seaweedClient struct {
@ -212,6 +215,42 @@ func (c *seaweedClient) ReleaseAdminToken(ctx context.Context, in *ReleaseAdminT
return out, nil
}
func (c *seaweedClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
out := new(PingResponse)
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error) {
out := new(RaftListClusterServersResponse)
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftListClusterServers", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error) {
out := new(RaftAddServerResponse)
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftAddServer", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) {
out := new(RaftRemoveServerResponse)
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftRemoveServer", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// SeaweedServer is the server API for Seaweed service.
// All implementations must embed UnimplementedSeaweedServer
// for forward compatibility
@ -230,6 +269,10 @@ type SeaweedServer interface {
ListClusterNodes(context.Context, *ListClusterNodesRequest) (*ListClusterNodesResponse, error)
LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error)
ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error)
Ping(context.Context, *PingRequest) (*PingResponse, error)
RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error)
RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error)
RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error)
mustEmbedUnimplementedSeaweedServer()
}
@ -279,6 +322,18 @@ func (UnimplementedSeaweedServer) LeaseAdminToken(context.Context, *LeaseAdminTo
func (UnimplementedSeaweedServer) ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReleaseAdminToken not implemented")
}
func (UnimplementedSeaweedServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedSeaweedServer) RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RaftListClusterServers not implemented")
}
func (UnimplementedSeaweedServer) RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RaftAddServer not implemented")
}
func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RaftRemoveServer not implemented")
}
func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {}
// UnsafeSeaweedServer may be embedded to opt out of forward compatibility for this service.
@ -560,6 +615,78 @@ func _Seaweed_ReleaseAdminToken_Handler(srv interface{}, ctx context.Context, de
return interceptor(ctx, in, info, handler)
}
func _Seaweed_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/master_pb.Seaweed/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).Ping(ctx, req.(*PingRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_RaftListClusterServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RaftListClusterServersRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).RaftListClusterServers(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/master_pb.Seaweed/RaftListClusterServers",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).RaftListClusterServers(ctx, req.(*RaftListClusterServersRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_RaftAddServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RaftAddServerRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).RaftAddServer(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/master_pb.Seaweed/RaftAddServer",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).RaftAddServer(ctx, req.(*RaftAddServerRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_RaftRemoveServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RaftRemoveServerRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).RaftRemoveServer(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/master_pb.Seaweed/RaftRemoveServer",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).RaftRemoveServer(ctx, req.(*RaftRemoveServerRequest))
}
return interceptor(ctx, in, info, handler)
}
// Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -615,6 +742,22 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{
MethodName: "ReleaseAdminToken",
Handler: _Seaweed_ReleaseAdminToken_Handler,
},
{
MethodName: "Ping",
Handler: _Seaweed_Ping_Handler,
},
{
MethodName: "RaftListClusterServers",
Handler: _Seaweed_RaftListClusterServers_Handler,
},
{
MethodName: "RaftAddServer",
Handler: _Seaweed_RaftAddServer_Handler,
},
{
MethodName: "RaftRemoveServer",
Handler: _Seaweed_RaftRemoveServer_Handler,
},
},
Streams: []grpc.StreamDesc{
{

4
weed/pb/messaging_pb/messaging.pb.go

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.19.4
// source: messaging.proto
package messaging_pb

1
weed/pb/messaging_pb/messaging_grpc.pb.go

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMessagingClient is the client API for SeaweedMessaging service.

25
weed/pb/mount.proto

@ -0,0 +1,25 @@
syntax = "proto3";
package messaging_pb;
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/mount_pb";
option java_package = "seaweedfs.client";
option java_outer_classname = "MountProto";
//////////////////////////////////////////////////
service SeaweedMount {
rpc Configure (ConfigureRequest) returns (ConfigureResponse) {
}
}
//////////////////////////////////////////////////
message ConfigureRequest {
int64 collection_capacity = 1;
}
message ConfigureResponse {
}

208
weed/pb/mount_pb/mount.pb.go

@ -0,0 +1,208 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc v3.19.4
// source: mount.proto
package mount_pb
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type ConfigureRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
CollectionCapacity int64 `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"`
}
func (x *ConfigureRequest) Reset() {
*x = ConfigureRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_mount_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ConfigureRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConfigureRequest) ProtoMessage() {}
func (x *ConfigureRequest) ProtoReflect() protoreflect.Message {
mi := &file_mount_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConfigureRequest.ProtoReflect.Descriptor instead.
func (*ConfigureRequest) Descriptor() ([]byte, []int) {
return file_mount_proto_rawDescGZIP(), []int{0}
}
func (x *ConfigureRequest) GetCollectionCapacity() int64 {
if x != nil {
return x.CollectionCapacity
}
return 0
}
type ConfigureResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *ConfigureResponse) Reset() {
*x = ConfigureResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_mount_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ConfigureResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConfigureResponse) ProtoMessage() {}
func (x *ConfigureResponse) ProtoReflect() protoreflect.Message {
mi := &file_mount_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConfigureResponse.ProtoReflect.Descriptor instead.
func (*ConfigureResponse) Descriptor() ([]byte, []int) {
return file_mount_proto_rawDescGZIP(), []int{1}
}
var File_mount_proto protoreflect.FileDescriptor
var file_mount_proto_rawDesc = []byte{
0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x43, 0x0a, 0x10, 0x43,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x61,
0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x63, 0x6f,
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79,
0x22, 0x13, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x5e, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x4e, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74,
0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77,
0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x6f,
0x75, 0x6e, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_mount_proto_rawDescOnce sync.Once
file_mount_proto_rawDescData = file_mount_proto_rawDesc
)
func file_mount_proto_rawDescGZIP() []byte {
file_mount_proto_rawDescOnce.Do(func() {
file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(file_mount_proto_rawDescData)
})
return file_mount_proto_rawDescData
}
var file_mount_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_mount_proto_goTypes = []interface{}{
(*ConfigureRequest)(nil), // 0: messaging_pb.ConfigureRequest
(*ConfigureResponse)(nil), // 1: messaging_pb.ConfigureResponse
}
var file_mount_proto_depIdxs = []int32{
0, // 0: messaging_pb.SeaweedMount.Configure:input_type -> messaging_pb.ConfigureRequest
1, // 1: messaging_pb.SeaweedMount.Configure:output_type -> messaging_pb.ConfigureResponse
1, // [1:2] is the sub-list for method output_type
0, // [0:1] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_mount_proto_init() }
func file_mount_proto_init() {
if File_mount_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_mount_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ConfigureRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_mount_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ConfigureResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_mount_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_mount_proto_goTypes,
DependencyIndexes: file_mount_proto_depIdxs,
MessageInfos: file_mount_proto_msgTypes,
}.Build()
File_mount_proto = out.File
file_mount_proto_rawDesc = nil
file_mount_proto_goTypes = nil
file_mount_proto_depIdxs = nil
}

100
weed/pb/mount_pb/mount_grpc.pb.go

@ -0,0 +1,100 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
package mount_pb
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMountClient is the client API for SeaweedMount service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type SeaweedMountClient interface {
Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error)
}
type seaweedMountClient struct {
cc grpc.ClientConnInterface
}
func NewSeaweedMountClient(cc grpc.ClientConnInterface) SeaweedMountClient {
return &seaweedMountClient{cc}
}
func (c *seaweedMountClient) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) {
out := new(ConfigureResponse)
err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMount/Configure", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// SeaweedMountServer is the server API for SeaweedMount service.
// All implementations must embed UnimplementedSeaweedMountServer
// for forward compatibility
type SeaweedMountServer interface {
Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error)
mustEmbedUnimplementedSeaweedMountServer()
}
// UnimplementedSeaweedMountServer must be embedded to have forward compatible implementations.
type UnimplementedSeaweedMountServer struct {
}
func (UnimplementedSeaweedMountServer) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
}
func (UnimplementedSeaweedMountServer) mustEmbedUnimplementedSeaweedMountServer() {}
// UnsafeSeaweedMountServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to SeaweedMountServer will
// result in compilation errors.
type UnsafeSeaweedMountServer interface {
mustEmbedUnimplementedSeaweedMountServer()
}
func RegisterSeaweedMountServer(s grpc.ServiceRegistrar, srv SeaweedMountServer) {
s.RegisterService(&SeaweedMount_ServiceDesc, srv)
}
func _SeaweedMount_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConfigureRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedMountServer).Configure(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/messaging_pb.SeaweedMount/Configure",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedMountServer).Configure(ctx, req.(*ConfigureRequest))
}
return interceptor(ctx, in, info, handler)
}
// SeaweedMount_ServiceDesc is the grpc.ServiceDesc for SeaweedMount service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var SeaweedMount_ServiceDesc = grpc.ServiceDesc{
ServiceName: "messaging_pb.SeaweedMount",
HandlerType: (*SeaweedMountServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Configure",
Handler: _SeaweedMount_Configure_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "mount.proto",
}

4
weed/pb/remote_pb/remote.pb.go

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.6.1
// protoc-gen-go v1.28.0
// protoc v3.19.4
// source: remote.proto
package remote_pb

15
weed/pb/server_address.go

@ -86,6 +86,14 @@ func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) {
return
}
func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) {
addresses = make(map[string]ServerAddress)
for _, address := range sa.ToAddresses() {
addresses[string(address)] = address
}
return
}
func (sa ServerAddresses) ToAddressStrings() (addresses []string) {
parts := strings.Split(string(sa), ",")
for _, address := range parts {
@ -101,6 +109,13 @@ func ToAddressStrings(addresses []ServerAddress) []string {
}
return strings
}
func ToAddressStringsFromMap(addresses map[string]ServerAddress) []string {
var strings []string
for _, addr := range addresses {
strings = append(strings, string(addr))
}
return strings
}
func FromAddressStrings(strings []string) []ServerAddress {
var addresses []ServerAddress
for _, addr := range strings {

14
weed/pb/volume_server.proto

@ -107,6 +107,10 @@ service VolumeServer {
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
}
rpc Ping (PingRequest) returns (PingResponse) {
}
}
//////////////////////////////////////////////////
@ -573,3 +577,13 @@ message VolumeNeedleStatusResponse {
uint32 crc = 5;
string ttl = 6;
}
message PingRequest {
string target = 1; // default to ping itself
string target_type = 2;
}
message PingResponse {
int64 start_time_ns = 1;
int64 remote_time_ns = 2;
int64 stop_time_ns = 3;
}

881
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File

37
weed/pb/volume_server_pb/volume_server_grpc.pb.go

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// VolumeServerClient is the client API for VolumeServer service.
@ -64,6 +63,7 @@ type VolumeServerClient interface {
// <experimental> query
Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error)
VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error)
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
}
type volumeServerClient struct {
@ -664,6 +664,15 @@ func (c *volumeServerClient) VolumeNeedleStatus(ctx context.Context, in *VolumeN
return out, nil
}
func (c *volumeServerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
out := new(PingResponse)
err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// VolumeServerServer is the server API for VolumeServer service.
// All implementations must embed UnimplementedVolumeServerServer
// for forward compatibility
@ -714,6 +723,7 @@ type VolumeServerServer interface {
// <experimental> query
Query(*QueryRequest, VolumeServer_QueryServer) error
VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error)
Ping(context.Context, *PingRequest) (*PingResponse, error)
mustEmbedUnimplementedVolumeServerServer()
}
@ -841,6 +851,9 @@ func (UnimplementedVolumeServerServer) Query(*QueryRequest, VolumeServer_QuerySe
func (UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented")
}
func (UnimplementedVolumeServerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {}
// UnsafeVolumeServerServer may be embedded to opt out of forward compatibility for this service.
@ -1604,6 +1617,24 @@ func _VolumeServer_VolumeNeedleStatus_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).Ping(ctx, req.(*PingRequest))
}
return interceptor(ctx, in, info, handler)
}
// VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -1731,6 +1762,10 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
MethodName: "VolumeNeedleStatus",
Handler: _VolumeServer_VolumeNeedleStatus_Handler,
},
{
MethodName: "Ping",
Handler: _VolumeServer_Ping_Handler,
},
},
Streams: []grpc.StreamDesc{
{

28
weed/remote_storage/remote_storage.go

@ -12,11 +12,11 @@ import (
"time"
)
const slash = "/"
func ParseLocationName(remote string) (locationName string) {
if strings.HasSuffix(string(remote), "/") {
remote = remote[:len(remote)-1]
}
parts := strings.SplitN(string(remote), "/", 2)
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 2)
if len(parts) >= 1 {
return parts[0]
}
@ -25,35 +25,31 @@ func ParseLocationName(remote string) (locationName string) {
func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
loc = &remote_pb.RemoteStorageLocation{}
if strings.HasSuffix(string(remote), "/") {
remote = remote[:len(remote)-1]
}
parts := strings.SplitN(string(remote), "/", 3)
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 3)
if len(parts) >= 1 {
loc.Name = parts[0]
}
if len(parts) >= 2 {
loc.Bucket = parts[1]
}
loc.Path = string(remote[len(loc.Name)+1+len(loc.Bucket):])
loc.Path = remote[len(loc.Name)+1+len(loc.Bucket):]
if loc.Path == "" {
loc.Path = "/"
loc.Path = slash
}
return
}
func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
loc = &remote_pb.RemoteStorageLocation{}
if strings.HasSuffix(string(remote), "/") {
remote = remote[:len(remote)-1]
}
parts := strings.SplitN(string(remote), "/", 2)
remote = strings.TrimSuffix(remote, slash)
parts := strings.SplitN(remote, slash, 2)
if len(parts) >= 1 {
loc.Name = parts[0]
}
loc.Path = string(remote[len(loc.Name):])
loc.Path = remote[len(loc.Name):]
if loc.Path == "" {
loc.Path = "/"
loc.Path = slash
}
return
}

27
weed/s3api/filer_multipart.go

@ -1,9 +1,11 @@
package s3api
import (
"encoding/hex"
"encoding/xml"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"golang.org/x/exp/slices"
"path/filepath"
"sort"
"strconv"
@ -12,7 +14,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/google/uuid"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -28,8 +29,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
glog.V(2).Infof("createMultipartUpload input %v", input)
uploadId, _ := uuid.NewRandom()
uploadIdString := uploadId.String()
uploadIdString := s3a.generateUploadID(*input.Key)
if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
@ -68,8 +68,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
glog.V(2).Infof("completeMultipartUpload input %v", input)
completedParts := parts.Parts
sort.Slice(completedParts, func(i, j int) bool {
return completedParts[i].PartNumber < completedParts[j].PartNumber
slices.SortFunc(completedParts, func(a, b CompletedPart) bool {
return a.PartNumber < b.PartNumber
})
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
@ -93,10 +93,15 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
for _, entry := range entries {
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
_, found := findByPartNumber(entry.Name, completedParts)
partETag, found := findByPartNumber(entry.Name, completedParts)
if !found {
continue
}
entryETag := hex.EncodeToString(entry.Attributes.GetMd5())
if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag {
glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag)
return nil, s3err.ErrInvalidPart
}
for _, chunk := range entry.Chunks {
p := &filer_pb.FileChunk{
FileId: chunk.GetFileIdString(),
@ -175,7 +180,15 @@ func findByPartNumber(fileName string, parts []CompletedPart) (etag string, foun
if parts[x].PartNumber != partNumber {
return
}
return parts[x].ETag, true
y := 0
for i, part := range parts[x:] {
if part.PartNumber == partNumber {
y = i
} else {
break
}
}
return parts[x+y].ETag, true
}
func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) {

6
weed/s3api/filer_multipart_test.go

@ -61,6 +61,10 @@ func Test_findByPartNumber(t *testing.T) {
ETag: "xxx",
PartNumber: 1,
},
CompletedPart{
ETag: "lll",
PartNumber: 1,
},
CompletedPart{
ETag: "yyy",
PartNumber: 3,
@ -83,7 +87,7 @@ func Test_findByPartNumber(t *testing.T) {
"0001.part",
parts,
},
"xxx",
"lll",
true,
},
{

20
weed/s3api/s3api_bucket_handlers.go

@ -3,6 +3,7 @@ package s3api
import (
"context"
"encoding/xml"
"errors"
"fmt"
"math"
"net/http"
@ -134,6 +135,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
w.Header().Set("Location", "/" + bucket)
writeSuccessResponseEmpty(w, r)
}
@ -148,6 +150,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
}
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
if !s3a.option.AllowDeleteBucketNotEmpty {
entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 1)
if err != nil {
return fmt.Errorf("failed to list bucket %s: %v", bucket, err)
}
if len(entries) > 0 {
return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code)
}
}
// delete collection
deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
@ -162,6 +173,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
return nil
})
if err != nil {
s3ErrorCode := s3err.ErrInternalError
if err.Error() == s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code {
s3ErrorCode = s3err.ErrBucketNotEmpty
}
s3err.WriteErrorResponse(w, r, s3ErrorCode)
return
}
err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)
if err != nil {

17
weed/s3api/s3api_object_handlers.go

@ -8,10 +8,10 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util/mem"
"golang.org/x/exp/slices"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
@ -27,6 +27,10 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
)
const (
deleteMultipleObjectsLimmit = 1000
)
func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser {
mimeBuffer := make([]byte, 512)
size, _ := dataReader.Read(mimeBuffer)
@ -59,7 +63,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
if r.Header.Get("Expires") != "" {
if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
s3err.WriteErrorResponse(w, r, s3err.ErrMalformedExpires)
return
}
}
@ -217,6 +221,11 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
return
}
if len(deleteObjects.Objects) > deleteMultipleObjectsLimmit {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxDeleteObjects)
return
}
var deletedObjects []ObjectIdentifier
var deleteErrors []DeleteError
var auditLog *s3err.AccessLog
@ -281,8 +290,8 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli
for dir, _ := range directoriesWithDeletion {
allDirs = append(allDirs, dir)
}
sort.Slice(allDirs, func(i, j int) bool {
return len(allDirs[i]) > len(allDirs[j])
slices.SortFunc(allDirs, func(a, b string) bool {
return len(a) > len(b)
})
newDirectoriesWithDeletion = make(map[string]int)
for _, dir := range allDirs {

44
weed/s3api/s3api_object_multipart_handlers.go

@ -2,6 +2,7 @@ package s3api
import (
"encoding/xml"
"crypto/sha1"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
@ -70,6 +71,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
// Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query())
err := s3a.checkUploadId(object, uploadID)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
@ -94,6 +100,11 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
// Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query())
err := s3a.checkUploadId(object, uploadID)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
Bucket: aws.String(bucket),
@ -165,6 +176,12 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
return
}
err := s3a.checkUploadId(object, uploadID)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
@ -186,11 +203,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
// PutObjectPartHandler - Put an object part in a multipart upload.
func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
bucket, _ := xhttp.GetBucketAndObject(r)
bucket, object := xhttp.GetBucketAndObject(r)
uploadID := r.URL.Query().Get("uploadId")
exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
if !exists {
err := s3a.checkUploadId(object, uploadID)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
@ -250,6 +267,27 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
}
// Generate uploadID hash string from object
func (s3a *S3ApiServer) generateUploadID(object string) string {
if strings.HasPrefix(object, "/") {
object = object[1:]
}
h := sha1.New()
h.Write([]byte(object))
return fmt.Sprintf("%x", h.Sum(nil))
}
//Check object name and uploadID when processing multipart uploading
func (s3a *S3ApiServer) checkUploadId(object string, id string) error {
hash := s3a.generateUploadID(object)
if hash != id {
glog.Errorf("object %s and uploadID %s are not matched", object, id)
return fmt.Errorf("object %s and uploadID %s are not matched", object, id)
}
return nil
}
// Parse bucket url queries for ?uploads
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
prefix = values.Get("prefix")

1
weed/s3api/s3api_server.go

@ -26,6 +26,7 @@ type S3ApiServerOption struct {
BucketsPath string
GrpcDialOption grpc.DialOption
AllowEmptyFolder bool
AllowDeleteBucketNotEmpty bool
LocalFilerSocket *string
}

6
weed/s3api/s3err/s3api_errors.go

@ -61,6 +61,7 @@ const (
ErrInvalidMaxKeys
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidMaxDeleteObjects
ErrInvalidPartNumberMarker
ErrInvalidPart
ErrInternalError
@ -157,6 +158,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Argument max-parts must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxDeleteObjects: {
Code: "InvalidArgument",
Description: "Argument objects can contain a list of up to 1000 keys",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidPartNumberMarker: {
Code: "InvalidArgument",
Description: "Argument partNumberMarker must be an integer.",

126
weed/server/filer_grpc_server.go

@ -3,7 +3,6 @@ package weed_server
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"os"
"path/filepath"
"strconv"
@ -357,128 +356,3 @@ func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.Delet
return &filer_pb.DeleteCollectionResponse{}, err
}
func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
var output *master_pb.StatisticsResponse
err = fs.filer.MasterClient.WithClient(false, func(masterClient master_pb.SeaweedClient) error {
grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{
Replication: req.Replication,
Collection: req.Collection,
Ttl: req.Ttl,
DiskType: req.DiskType,
})
if grpcErr != nil {
return grpcErr
}
output = grpcResponse
return nil
})
if err != nil {
return nil, err
}
return &filer_pb.StatisticsResponse{
TotalSize: output.TotalSize,
UsedSize: output.UsedSize,
FileCount: output.FileCount,
}, nil
}
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
t := &filer_pb.GetFilerConfigurationResponse{
Masters: pb.ToAddressStrings(fs.option.Masters),
Collection: fs.option.Collection,
Replication: fs.option.DefaultReplication,
MaxMb: uint32(fs.option.MaxMB),
DirBuckets: fs.filer.DirBucketsPath,
Cipher: fs.filer.Cipher,
Signature: fs.filer.Signature,
MetricsAddress: fs.metricsAddress,
MetricsIntervalSec: int32(fs.metricsIntervalSec),
Version: util.Version(),
ClusterId: string(clusterId),
}
glog.V(4).Infof("GetFilerConfiguration: %v", t)
return t, nil
}
func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error {
req, err := stream.Recv()
if err != nil {
return err
}
clientName := util.JoinHostPort(req.Name, int(req.GrpcPort))
m := make(map[string]bool)
for _, tp := range req.Resources {
m[tp] = true
}
fs.brokersLock.Lock()
fs.brokers[clientName] = m
glog.V(0).Infof("+ broker %v", clientName)
fs.brokersLock.Unlock()
defer func() {
fs.brokersLock.Lock()
delete(fs.brokers, clientName)
glog.V(0).Infof("- broker %v: %v", clientName, err)
fs.brokersLock.Unlock()
}()
for {
if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil {
glog.V(0).Infof("send broker %v: %+v", clientName, err)
return err
}
// println("replied")
if _, err := stream.Recv(); err != nil {
glog.V(0).Infof("recv broker %v: %v", clientName, err)
return err
}
// println("received")
}
}
func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) {
resp = &filer_pb.LocateBrokerResponse{}
fs.brokersLock.Lock()
defer fs.brokersLock.Unlock()
var localBrokers []*filer_pb.LocateBrokerResponse_Resource
for b, m := range fs.brokers {
if _, found := m[req.Resource]; found {
resp.Found = true
resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{
{
GrpcAddresses: b,
ResourceCount: int32(len(m)),
},
}
return
}
localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{
GrpcAddresses: b,
ResourceCount: int32(len(m)),
})
}
resp.Resources = localBrokers
return resp, nil
}

177
weed/server/filer_grpc_server_admin.go

@ -0,0 +1,177 @@
package weed_server
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"time"
)
func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
var output *master_pb.StatisticsResponse
err = fs.filer.MasterClient.WithClient(false, func(masterClient master_pb.SeaweedClient) error {
grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{
Replication: req.Replication,
Collection: req.Collection,
Ttl: req.Ttl,
DiskType: req.DiskType,
})
if grpcErr != nil {
return grpcErr
}
output = grpcResponse
return nil
})
if err != nil {
return nil, err
}
return &filer_pb.StatisticsResponse{
TotalSize: output.TotalSize,
UsedSize: output.UsedSize,
FileCount: output.FileCount,
}, nil
}
func (fs *FilerServer) Ping(ctx context.Context, req *filer_pb.PingRequest) (resp *filer_pb.PingResponse, pingErr error) {
resp = &filer_pb.PingResponse{
StartTimeNs: time.Now().UnixNano(),
}
if req.TargetType == cluster.FilerType {
pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.VolumeServerType {
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.MasterType {
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client master_pb.SeaweedClient) error {
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if pingErr != nil {
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
}
resp.StopTimeNs = time.Now().UnixNano()
return
}
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
t := &filer_pb.GetFilerConfigurationResponse{
Masters: pb.ToAddressStringsFromMap(fs.option.Masters),
Collection: fs.option.Collection,
Replication: fs.option.DefaultReplication,
MaxMb: uint32(fs.option.MaxMB),
DirBuckets: fs.filer.DirBucketsPath,
Cipher: fs.filer.Cipher,
Signature: fs.filer.Signature,
MetricsAddress: fs.metricsAddress,
MetricsIntervalSec: int32(fs.metricsIntervalSec),
Version: util.Version(),
ClusterId: string(clusterId),
}
glog.V(4).Infof("GetFilerConfiguration: %v", t)
return t, nil
}
func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error {
req, err := stream.Recv()
if err != nil {
return err
}
clientName := util.JoinHostPort(req.Name, int(req.GrpcPort))
m := make(map[string]bool)
for _, tp := range req.Resources {
m[tp] = true
}
fs.brokersLock.Lock()
fs.brokers[clientName] = m
glog.V(0).Infof("+ broker %v", clientName)
fs.brokersLock.Unlock()
defer func() {
fs.brokersLock.Lock()
delete(fs.brokers, clientName)
glog.V(0).Infof("- broker %v: %v", clientName, err)
fs.brokersLock.Unlock()
}()
for {
if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil {
glog.V(0).Infof("send broker %v: %+v", clientName, err)
return err
}
// println("replied")
if _, err := stream.Recv(); err != nil {
glog.V(0).Infof("recv broker %v: %v", clientName, err)
return err
}
// println("received")
}
}
func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) {
resp = &filer_pb.LocateBrokerResponse{}
fs.brokersLock.Lock()
defer fs.brokersLock.Unlock()
var localBrokers []*filer_pb.LocateBrokerResponse_Resource
for b, m := range fs.brokers {
if _, found := m[req.Resource]; found {
resp.Found = true
resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{
{
GrpcAddresses: b,
ResourceCount: int32(len(m)),
},
}
return
}
localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{
GrpcAddresses: b,
ResourceCount: int32(len(m)),
})
}
resp.Resources = localBrokers
return resp, nil
}

2
weed/server/filer_server.go

@ -48,7 +48,7 @@ import (
)
type FilerOption struct {
Masters []pb.ServerAddress
Masters map[string]pb.ServerAddress
Collection string
DefaultReplication string
DisableDirListing bool

6
weed/server/filer_server_handlers_read_dir.go

@ -46,8 +46,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
path = ""
}
emptyFolder := true
if len(entries) > 0 {
lastFileName = entries[len(entries)-1].Name()
emptyFolder = false
}
glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries))
@ -59,12 +61,14 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
Limit int
LastFileName string
ShouldDisplayLoadMore bool
EmptyFolder bool
}{
path,
entries,
limit,
lastFileName,
shouldDisplayLoadMore,
emptyFolder,
})
return
}
@ -76,6 +80,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
Limit int
LastFileName string
ShouldDisplayLoadMore bool
EmptyFolder bool
}{
path,
ui.ToBreadcrumb(path),
@ -83,5 +88,6 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
limit,
lastFileName,
shouldDisplayLoadMore,
emptyFolder,
})
}

2
weed/server/filer_server_handlers_tagging.go

@ -82,8 +82,10 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque
toDelete := strings.Split(r.URL.Query().Get("tagging"), ",")
deletions := make(map[string]struct{})
for _, deletion := range toDelete {
if deletion != "" {
deletions[deletion] = struct{}{}
}
}
// delete all tags or specific tags
hasDeletion := false

12
weed/server/filer_server_handlers_write_autochunk.go

@ -164,6 +164,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
}
var entry *filer.Entry
var newChunks []*filer_pb.FileChunk
var mergedChunks []*filer_pb.FileChunk
isAppend := isAppend(r)
@ -186,7 +187,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
}
entry.FileSize += uint64(chunkOffset)
}
mergedChunks = append(entry.Chunks, fileChunks...)
newChunks = append(entry.Chunks, fileChunks...)
// TODO
if len(entry.Content) > 0 {
@ -196,7 +197,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
} else {
glog.V(4).Infoln("saving", path)
mergedChunks = fileChunks
newChunks = fileChunks
entry = &filer.Entry{
FullPath: util.FullPath(path),
Attr: filer.Attr{
@ -217,6 +218,13 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
}
}
// maybe concatenate small chunks into one whole chunk
mergedChunks, replyerr = fs.maybeMergeChunks(so, newChunks)
if replyerr != nil {
glog.V(0).Infof("merge chunks %s: %v", r.RequestURI, replyerr)
mergedChunks = newChunks
}
// maybe compact entry chunks
mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks)
if replyerr != nil {

11
weed/server/filer_server_handlers_write_merge.go

@ -0,0 +1,11 @@
package weed_server
import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func (fs *FilerServer) maybeMergeChunks(so *operation.StorageOption, inputChunks []*filer_pb.FileChunk) (mergedChunks []*filer_pb.FileChunk, err error) {
//TODO merge consecutive smaller chunks into a large chunk to reduce number of chunks
return inputChunks, nil
}

8
weed/server/filer_server_handlers_write_upload.go

@ -4,10 +4,10 @@ import (
"bytes"
"crypto/md5"
"fmt"
"golang.org/x/exp/slices"
"hash"
"io"
"net/http"
"sort"
"strconv"
"strings"
"sync"
@ -130,11 +130,9 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
fs.filer.DeleteChunks(fileChunks)
return nil, md5Hash, 0, uploadErr, nil
}
sort.Slice(fileChunks, func(i, j int) bool {
return fileChunks[i].Offset < fileChunks[j].Offset
slices.SortFunc(fileChunks, func(a, b *filer_pb.FileChunk) bool {
return a.Offset < b.Offset
})
return fileChunks, md5Hash, chunkOffset, nil, smallContent
}

6
weed/server/filer_ui/breadcrumb.go

@ -15,8 +15,12 @@ func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) {
parts := strings.Split(fullpath, "/")
for i := 0; i < len(parts); i++ {
name := parts[i]
if name == "" {
name = "/"
}
crumb := Breadcrumb{
Name: parts[i] + " /",
Name: name,
Link: "/" + util.Join(parts[0:i+1]...),
}
if !strings.HasSuffix(crumb.Link, "/") {

247
weed/server/filer_ui/filer.html

@ -11,6 +11,7 @@
#drop-area {
border: 1px transparent;
margin-top: 5px;
}
#drop-area.highlight {
@ -26,6 +27,12 @@
border-radius: 2px;
border: 1px solid #ccc;
float: right;
margin-left: 2px;
margin-bottom: 0;
}
label {
font-weight: normal;
}
.button:hover {
@ -36,6 +43,38 @@
display: none;
}
td, th {
vertical-align: bottom;
}
.table-hover > tbody > tr:hover > * > div.operations {
display: block;
}
.table > tbody > tr {
height: 39px;
}
div.operations {
display: none;
}
.footer {
position: absolute;
bottom: 0px;
right: 5%;
min-width: 25%;
border-left: 1px solid #ccc;
border-right: 1px solid #ccc;
}
.add-files {
font-size: 46px;
text-align: center;
border: 1px dashed #999;
padding-bottom: 9px;
margin: 0 2px;
}
</style>
</head>
<body>
@ -48,12 +87,21 @@
</div>
<div class="row">
<div>
<div class="btn-group btn-group-sm pull-right" role="group" style="margin-top:3px;">
<label class="btn btn-default" onclick="handleCreateDir()">
<span class="glyphicon glyphicon-plus" aria-hidden="true"></span> New Folder
</label>
<label class="btn btn-default" for="fileElem">
<span class="glyphicon glyphicon-cloud-upload" aria-hidden="true"></span> Upload
</label>
</div>
<ol class="breadcrumb">
{{ range $entry := .Breadcrumbs }}
<a href="{{ printpath $entry.Link }}">
<li><a href="{{ printpath $entry.Link }}">
{{ $entry.Name }}
</a>
</li></a>
{{ end }}
<label class="button" for="fileElem">Upload</label>
</ol>
</div>
</div>
@ -61,13 +109,18 @@
<form class="upload-form">
<input type="file" id="fileElem" multiple onchange="handleFiles(this.files)">
<table width="90%">
{{if .EmptyFolder}}
<div class="row add-files">
+
</div>
{{else}}
<table width="100%" class="table table-hover">
{{$path := .Path }}
{{ range $entry_index, $entry := .Entries }}
<tr>
<td>
{{if $entry.IsDirectory}}
<img src="/seaweedfsstatic/images/folder.gif" width="20" height="23">
<span class="glyphicon glyphicon-folder-open" aria-hidden="true"></span>&nbsp;
<a href="{{ printpath $path "/" $entry.Name "/"}}" >
{{ $entry.Name }}
</a>
@ -89,13 +142,29 @@
{{ $entry.Size | humanizeBytes }}&nbsp;
{{end}}
</td>
<td nowrap>
<td align="right" nowrap>
{{ $entry.Timestamp.Format "2006-01-02 15:04" }}
</td>
<td style="width:75px">
<div class="btn-group btn-group-xs pull-right operations" role="group">
<label class="btn" onclick="handleRename('{{ $entry.Name }}', '{{ printpath $path "/" }}')">
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
</label>
{{if $entry.IsDirectory}}
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</label>
{{else}}
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name }}')">
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</label>
{{end}}
</div>
</td>
</tr>
{{ end }}
</table>
{{end}}
</form>
</div>
@ -109,65 +178,177 @@
<br/>
<br/>
<div id="progress-area" class="footer" style="display: none;">
</div>
</div>
</body>
<script type="text/javascript">
// ************************ Drag and drop ***************** //
let dropArea = document.getElementById("drop-area")
let dropArea = document.getElementById("drop-area");
let progressArea = document.getElementById("progress-area");
// Prevent default drag behaviors
;['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
dropArea.addEventListener(eventName, preventDefaults, false)
document.body.addEventListener(eventName, preventDefaults, false)
})
dropArea.addEventListener(eventName, preventDefaults, false);
document.body.addEventListener(eventName, preventDefaults, false);
});
// Highlight drop area when item is dragged over it
;['dragenter', 'dragover'].forEach(eventName => {
dropArea.addEventListener(eventName, highlight, false)
})
dropArea.addEventListener(eventName, highlight, false);
});
;['dragleave', 'drop'].forEach(eventName => {
dropArea.addEventListener(eventName, unhighlight, false)
})
dropArea.addEventListener(eventName, unhighlight, false);
});
// Handle dropped files
dropArea.addEventListener('drop', handleDrop, false)
dropArea.addEventListener('drop', handleDrop, false);
function preventDefaults(e) {
e.preventDefault()
e.stopPropagation()
e.preventDefault();
e.stopPropagation();
}
function highlight(e) {
dropArea.classList.add('highlight')
dropArea.classList.add('highlight');
}
function unhighlight(e) {
dropArea.classList.remove('highlight')
dropArea.classList.remove('highlight');
}
function handleDrop(e) {
var dt = e.dataTransfer
var files = dt.files
var dt = e.dataTransfer;
var files = dt.files;
handleFiles(files)
handleFiles(files);
}
var uploadList = {};
function handleFiles(files) {
files = [...files]
files.forEach(uploadFile)
window.location.reload()
files = [...files];
files.forEach(startUpload);
renderProgress();
files.forEach(uploadFile);
}
function startUpload(file, i) {
uploadList[file.name] = {'name': file.name, 'percent': 0, 'finish': false};
}
function renderProgress() {
var values = Object.values(uploadList);
var html = '<table class="table">\n<tr><th>Uploading</th><\/tr>\n';
for (let i of values) {
var progressBarClass = 'progress-bar-striped active';
if (i.percent >= 100) {
progressBarClass = 'progress-bar-success';
}
html += '<tr>\n<td>\n';
html += '<div class="progress" style="margin-bottom: 2px;">\n';
html += '<div class="progress-bar ' + progressBarClass + '" role="progressbar" aria-valuenow="' + '100" aria-valuemin="0" aria-valuemax="100" style="width:' + i.percent + '%;">';
html += '<span style="margin-right: 10px;">' + i.name + '</span>' + i.percent + '%<\/div>';
html += '<\/div>\n<\/td>\n<\/tr>\n';
}
html += '<\/table>\n';
progressArea.innerHTML = html;
if (values.length > 0) {
progressArea.attributes.style.value = '';
}
}
function reportProgress(file, percent) {
var item = uploadList[file]
item.percent = percent;
renderProgress();
}
function finishUpload(file) {
uploadList[file]['finish'] = true;
renderProgress();
var allFinish = true;
for (let i of Object.values(uploadList)) {
if (!i.finish) {
allFinish = false;
break;
}
}
if (allFinish) {
console.log('All Finish');
window.location.reload();
}
}
function uploadFile(file, i) {
var url = window.location.href
var xhr = new XMLHttpRequest()
var formData = new FormData()
xhr.open('POST', url, false)
var url = window.location.href;
var xhr = new XMLHttpRequest();
var fileName = file.name;
xhr.upload.addEventListener('progress', function(e) {
if (e.lengthComputable) {
var percent = Math.ceil((e.loaded / e.total) * 100);
reportProgress(fileName, percent)
}
});
xhr.upload.addEventListener('loadend', function(e) {
finishUpload(fileName);
});
var formData = new FormData();
xhr.open('POST', url, true);
formData.append('file', file);
xhr.send(formData);
}
function handleCreateDir() {
var dirName = prompt('Folder Name:', '');
dirName = dirName.trim();
if (dirName == null || dirName == '') {
return;
}
var baseUrl = window.location.href;
if (!baseUrl.endsWith('/')) {
baseUrl += '/';
}
var url = baseUrl + dirName;
if (!url.endsWith('/')) {
url += '/';
}
var xhr = new XMLHttpRequest();
xhr.open('POST', url, false);
xhr.setRequestHeader('Content-Type', '');
xhr.send();
window.location.reload();
}
function handleRename(originName, basePath) {
var newName = prompt('New Name:', originName);
if (newName == null || newName == '') {
return;
}
var url = basePath + newName;
var originPath = basePath + originName;
url += '?mv.from=' + originPath;
var xhr = new XMLHttpRequest();
xhr.open('POST', url, false);
xhr.setRequestHeader('Content-Type', '');
xhr.send();
window.location.reload();
}
function handleDelete(path) {
if (!confirm('Are you sure to delete ' + path + '?')) {
return;
}
var url = path;
if (url.endsWith('/')) {
url += '?recursive=true';
}
formData.append('file', file)
xhr.send(formData)
var xhr = new XMLHttpRequest();
xhr.open('DELETE', url, false);
xhr.send();
window.location.reload();
}
</script>
</html>

15
weed/server/master_grpc_server.go

@ -113,6 +113,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes {
dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
ms.Topo.DataNodeRegistration(dcName, rackName, dn)
// process heartbeat.Volumes
stats.MasterReceivedHeartbeatCounter.WithLabelValues("Volumes").Inc()
newVolumes, deletedVolumes := ms.Topo.SyncDataNodeRegistration(heartbeat.Volumes, dn)
@ -133,13 +136,13 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
ms.Topo.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn)
for _, s := range heartbeat.NewEcShards {
message.NewVids = append(message.NewVids, s.Id)
message.NewEcVids = append(message.NewEcVids, s.Id)
}
for _, s := range heartbeat.DeletedEcShards {
if dn.HasVolumesById(needle.VolumeId(s.Id)) {
if dn.HasEcShards(needle.VolumeId(s.Id)) {
continue
}
message.DeletedVids = append(message.DeletedVids, s.Id)
message.DeletedEcVids = append(message.DeletedEcVids, s.Id)
}
}
@ -151,17 +154,17 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
// broadcast the ec vid changes to master clients
for _, s := range newShards {
message.NewVids = append(message.NewVids, uint32(s.VolumeId))
message.NewEcVids = append(message.NewEcVids, uint32(s.VolumeId))
}
for _, s := range deletedShards {
if dn.HasVolumesById(s.VolumeId) {
continue
}
message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
message.DeletedEcVids = append(message.DeletedEcVids, uint32(s.VolumeId))
}
}
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 || len(message.NewEcVids) > 0 || len(message.DeletedEcVids) > 0 {
ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message})
}

42
weed/server/master_grpc_server_admin.go

@ -3,7 +3,11 @@ package weed_server
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"math/rand"
"sync"
"time"
@ -142,3 +146,41 @@ func (ms *MasterServer) ReleaseAdminToken(ctx context.Context, req *master_pb.Re
}
return resp, nil
}
func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (resp *master_pb.PingResponse, pingErr error) {
resp = &master_pb.PingResponse{
StartTimeNs: time.Now().UnixNano(),
}
if req.TargetType == cluster.FilerType {
pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.VolumeServerType {
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.MasterType {
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client master_pb.SeaweedClient) error {
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if pingErr != nil {
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
}
resp.StopTimeNs = time.Now().UnixNano()
return
}

66
weed/server/master_grpc_server_raft.go

@ -0,0 +1,66 @@
package weed_server
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/hashicorp/raft"
)
func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_pb.RaftListClusterServersRequest) (*master_pb.RaftListClusterServersResponse, error) {
resp := &master_pb.RaftListClusterServersResponse{}
servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
for _, server := range servers {
resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{
Id: string(server.ID),
Address: string(server.Address),
Suffrage: server.Suffrage.String(),
})
}
return resp, nil
}
func (ms *MasterServer) RaftAddServer(ctx context.Context, req *master_pb.RaftAddServerRequest) (*master_pb.RaftAddServerResponse, error) {
resp := &master_pb.RaftAddServerResponse{}
if ms.Topo.HashicorpRaft.State() != raft.Leader {
return nil, fmt.Errorf("raft add server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
}
var idxFuture raft.IndexFuture
if req.Voter {
idxFuture = ms.Topo.HashicorpRaft.AddVoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
} else {
idxFuture = ms.Topo.HashicorpRaft.AddNonvoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
}
if err := idxFuture.Error(); err != nil {
return nil, err
}
return resp, nil
}
func (ms *MasterServer) RaftRemoveServer(ctx context.Context, req *master_pb.RaftRemoveServerRequest) (*master_pb.RaftRemoveServerResponse, error) {
resp := &master_pb.RaftRemoveServerResponse{}
if ms.Topo.HashicorpRaft.State() != raft.Leader {
return nil, fmt.Errorf("raft remove server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
}
if !req.Force {
ms.clientChansLock.RLock()
_, ok := ms.clientChans[fmt.Sprintf("%s@%s", cluster.MasterType, req.Id)]
ms.clientChansLock.RUnlock()
if ok {
return resp, fmt.Errorf("raft remove server %s failed: client connection to master exists", req.Id)
}
}
idxFuture := ms.Topo.HashicorpRaft.RemoveServer(raft.ServerID(req.Id), 0, 0)
if err := idxFuture.Error(); err != nil {
return nil, err
}
return resp, nil
}

2
weed/server/master_grpc_server_volume.go

@ -268,7 +268,7 @@ func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumV
resp := &master_pb.VacuumVolumeResponse{}
ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.preallocateSize)
ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), req.VolumeId, req.Collection, ms.preallocateSize)
return resp, nil
}

114
weed/server/master_server.go

@ -1,6 +1,7 @@
package weed_server
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/stats"
"net/http"
@ -17,6 +18,7 @@ import (
"github.com/chrislusf/raft"
"github.com/gorilla/mux"
hashicorpRaft "github.com/hashicorp/raft"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -32,6 +34,7 @@ import (
const (
SequencerType = "master.sequencer.type"
SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
RaftServerRemovalTime = 72 * time.Minute
)
type MasterOption struct {
@ -62,6 +65,9 @@ type MasterServer struct {
boundedLeaderChan chan int
onPeerUpdatDoneCn chan string
onPeerUpdatDoneCnExist bool
// notifying clients
clientChansLock sync.RWMutex
clientChans map[string]chan *master_pb.KeepConnectedResponse
@ -75,7 +81,7 @@ type MasterServer struct {
Cluster *cluster.Cluster
}
func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
v := util.GetViper()
signingKey := v.GetString("jwt.signing.key")
@ -112,6 +118,9 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddre
Cluster: cluster.NewCluster(),
}
ms.boundedLeaderChan = make(chan int, 16)
ms.onPeerUpdatDoneCn = make(chan string)
ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
seq := ms.createSequencer(option)
if nil == seq {
@ -160,6 +169,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddre
}
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
var raftServerName string
if raftServer.raftServer != nil {
ms.Topo.RaftServer = raftServer.raftServer
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
@ -168,11 +179,31 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
}
})
raftServerName = ms.Topo.RaftServer.Name()
} else if raftServer.RaftHashicorp != nil {
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
leaderCh := raftServer.RaftHashicorp.LeaderCh()
prevLeader := ms.Topo.HashicorpRaft.Leader()
go func() {
for {
select {
case isLeader := <-leaderCh:
leader := ms.Topo.HashicorpRaft.Leader()
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
prevLeader = leader
}
}
}()
raftServerName = ms.Topo.HashicorpRaft.String()
}
if ms.Topo.IsLeader() {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
} else {
if ms.Topo.RaftServer.Leader() != "" {
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
}
}
}
@ -181,16 +212,27 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if ms.Topo.IsLeader() {
f(w, r)
} else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
return
}
var raftServerLeader string
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
raftServerLeader = ms.Topo.RaftServer.Leader()
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
raftServerLeader = string(ms.Topo.HashicorpRaft.Leader())
}
if raftServerLeader == "" {
f(w, r)
return
}
ms.boundedLeaderChan <- 1
defer func() { <-ms.boundedLeaderChan }()
targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
targetUrl, err := url.Parse("http://" + raftServerLeader)
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError,
fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
return
}
glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
glog.V(4).Infoln("proxying to leader", raftServerLeader)
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
director := proxy.Director
proxy.Director = func(req *http.Request) {
@ -202,10 +244,6 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
}
proxy.Transport = util.Transport
proxy.ServeHTTP(w, r)
} else {
// handle requests locally
f(w, r)
}
}
}
@ -301,3 +339,57 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
}
return seq
}
func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
return
}
glog.V(4).Infof("OnPeerUpdate: %+v", update)
peerAddress := pb.ServerAddress(update.Address)
peerName := string(peerAddress)
isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
if update.IsAdd {
if isLeader {
raftServerFound := false
for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
if string(server.ID) == peerName {
raftServerFound = true
}
}
if !raftServerFound {
glog.V(0).Infof("adding new raft server: %s", peerName)
ms.Topo.HashicorpRaft.AddVoter(
hashicorpRaft.ServerID(peerName),
hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
}
}
if ms.onPeerUpdatDoneCnExist {
ms.onPeerUpdatDoneCn <- peerName
}
} else if isLeader {
go func(peerName string) {
for {
select {
case <-time.After(RaftServerRemovalTime):
err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
Id: peerName,
Force: false,
})
return err
})
if err != nil {
glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
}
return
case peerDone := <-ms.onPeerUpdatDoneCn:
if peerName == peerDone {
return
}
}
}
}(peerName)
ms.onPeerUpdatDoneCnExist = true
}
}

2
weed/server/master_server_handlers_admin.go

@ -64,7 +64,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
}
}
// glog.Infoln("garbageThreshold =", gcThreshold)
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, 0, "", ms.preallocateSize)
ms.dirStatusHandler(w, r)
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save