diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index df2f9fed8..2c298cf4e 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -36,7 +36,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 6f103576e..ad6cb0861 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index c63135f38..d9da49920 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index e3682f463..612898014 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 41330815e..912045355 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 901fe0bc4..7fb6d602c 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index bc499dc7c..6e8dc4ccb 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index ad69107d9..b21259656 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - name: 'Dependency Review' - uses: actions/dependency-review-action@733dd5d4a5203f238c33806593ec0f5fc5343d8c + uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml new file mode 100644 index 000000000..fe6ce4a80 --- /dev/null +++ b/.github/workflows/s3tests.yml @@ -0,0 +1,190 @@ +name: "Ceph S3 tests" + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.head_ref }}/s3tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: docker + +jobs: + s3tests: + name: Ceph S3 tests + runs-on: ubuntu-22.04 + container: + image: docker.io/kmlebedev/ceph-s3-tests:0.0.2 + timeout-minutes: 30 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + id: go + + - name: Run Ceph S3 tests + timeout-minutes: 15 + env: + S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf + shell: bash + run: | + cd /__w/seaweedfs/seaweedfs/weed + go install -buildvcs=false + set -x + nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + sleep 10 + cd /s3-tests + tox -- \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \ + s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \ + s3tests_boto3/functional/test_s3.py::test_multi_object_delete \ + s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \ + s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \ + s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \ + s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \ + s3tests_boto3/functional/test_s3.py::test_object_write_expires \ + s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \ + s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \ + s3tests_boto3/functional/test_s3.py::test_object_write_file \ + s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \ + s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \ + s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \ + s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \ + s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \ + s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \ + s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \ + s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests_boto3/functional/test_s3.py::test_bucket_head \ + s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \ + s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \ + s3tests_boto3/functional/test_s3.py::test_object_anon_put \ + s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \ + s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \ + s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \ + s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \ + s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \ + s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \ + s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \ + s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \ + s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \ + s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \ + s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \ + s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \ + s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \ + s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \ + s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ + s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \ + s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed diff --git a/Makefile b/Makefile index 269b6ec91..869de2f47 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone" server: install - weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324 + weed -v 0 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324 benchmark: install warp_install pkill weed || true diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf index 68d9ddeb7..f8d5b4930 100644 --- a/docker/compose/s3tests.conf +++ b/docker/compose/s3tests.conf @@ -2,7 +2,7 @@ ## this section is just used for host, port and bucket_prefix # host set for rgw in vstart.sh -host = s3 +host = 127.0.0.1 # port set for rgw in vstart.sh port = 8000 diff --git a/go.mod b/go.mod index 77d1d0167..5d75ab429 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.50.36 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 - github.com/cenkalti/backoff/v4 v4.2.1 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -30,10 +30,10 @@ require ( github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/fclairamb/ftpserverlib v0.24.0 github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-errors/errors v1.4.2 // indirect + github.com/go-errors/errors v1.5.1 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.11.0 - github.com/go-sql-driver/mysql v1.8.0 + github.com/go-sql-driver/mysql v1.8.1 github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect @@ -43,7 +43,7 @@ require ( github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 // indirect - github.com/googleapis/gax-go/v2 v2.12.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gorilla/mux v1.8.1 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -57,11 +57,11 @@ require ( github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/reedsolomon v1.12.1 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 - github.com/linxGnu/grocksdb v1.8.12 + github.com/linxGnu/grocksdb v1.8.14 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect @@ -96,9 +96,8 @@ require ( github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 github.com/valyala/bytebufferpool v1.0.0 - github.com/viant/assertly v0.5.4 // indirect - github.com/viant/ptrie v0.3.0 - github.com/viant/toolbox v0.33.2 // indirect + github.com/viant/ptrie v0.3.1 + github.com/viant/toolbox v0.34.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -107,18 +106,18 @@ require ( go.mongodb.org/mongo-driver v1.14.0 go.opencensus.io v0.24.0 // indirect gocloud.dev v0.37.0 - gocloud.dev/pubsub/natspubsub v0.36.0 + gocloud.dev/pubsub/natspubsub v0.37.0 gocloud.dev/pubsub/rabbitpubsub v0.36.0 golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 golang.org/x/image v0.15.0 golang.org/x/net v0.22.0 golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.169.0 + google.golang.org/api v0.172.0 google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/grpc v1.62.1 @@ -140,9 +139,9 @@ require ( github.com/Jille/raft-grpc-transport v1.5.0 github.com/arangodb/go-driver v1.6.1 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.26.0 - github.com/aws/aws-sdk-go-v2/config v1.27.7 - github.com/aws/aws-sdk-go-v2/credentials v1.17.7 + github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 github.com/fluent/fluent-logger-golang v1.9.0 github.com/getsentry/sentry-go v0.27.0 @@ -155,12 +154,12 @@ require ( github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.66.0 github.com/rdleal/intervalst v1.3.0 - github.com/schollz/progressbar/v3 v3.14.1 - github.com/shirou/gopsutil/v3 v3.24.1 + github.com/schollz/progressbar/v3 v3.14.2 + github.com/shirou/gopsutil/v3 v3.24.3 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 - github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1 - go.etcd.io/etcd/client/pkg/v3 v3.5.12 + github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0 + go.etcd.io/etcd/client/pkg/v3 v3.5.13 go.uber.org/atomic v1.11.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) @@ -191,21 +190,21 @@ require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect - github.com/aws/smithy-go v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bradenaw/juniper v0.15.2 // indirect github.com/buengese/sgzip v0.1.1 // indirect @@ -267,8 +266,8 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/montanaflynn/stats v0.7.0 // indirect - github.com/nats-io/nats.go v1.31.0 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/nats.go v1.33.1 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/ncw/swift/v2 v2.0.2 // indirect github.com/onsi/gomega v1.27.10 // indirect @@ -290,7 +289,7 @@ require ( github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect github.com/rfjakob/eme v1.1.2 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -314,7 +313,7 @@ require ( github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect go.etcd.io/bbolt v1.3.8 // indirect @@ -331,7 +330,7 @@ require ( golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index fd7701849..180cb8b4f 100644 --- a/go.sum +++ b/go.sum @@ -138,30 +138,30 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aws/aws-sdk-go v1.50.36 h1:PjWXHwZPuTLMR1NIb8nEjLucZBMzmf84TLoLbD8BZqk= github.com/aws/aws-sdk-go v1.50.36/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= -github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= -github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4= -github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 h1:mDnFOE2sVkyphMWtTH+stv0eW3k0OTx94K63xpxHty4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3/go.mod h1:V8MuRVcCRt5h1S+Fwu8KbC7l/gBGo3yBAyUbJM2IJOk= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 h1:mbWNpfRUTT6bnacmvOTKXZjR/HycibdWzNpfbrbLDIs= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5/go.mod h1:FCOPWGjsshkkICJIn9hq9xr6dLKtyaWpuUojiN3W1/8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 h1:4t+QEX7BsXz98W8W1lNvMAG+NX8qHz2CjLBxQKku40g= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3/go.mod h1:oFcjjUq5Hm09N9rpxTdeMeLeQcxS7mIkBkL8qUKng+A= github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 h1:lW5xUzOPGAMY7HPuNF4FdyBwRc3UJ/e8KsapbesVeNU= @@ -170,14 +170,14 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMz github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2/go.mod h1:J3XhTE+VsY1jDsdDY+ACFAppZj/gpvygzC5JE0bTLbQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8= -github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= -github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -201,8 +201,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -330,8 +330,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -367,8 +367,8 @@ github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4 github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= -github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4= -github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -486,8 +486,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= -github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -595,8 +595,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -627,8 +627,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.8.12 h1:1/pCztQUOa3BX/1gR3jSZDoaKFpeHFvQ1XrqZpSvZVo= -github.com/linxGnu/grocksdb v1.8.12/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.14 h1:HTgyYalNwBSG/1qCQUIott44wU5b2Y9Kr3z7SK5OfGQ= +github.com/linxGnu/grocksdb v1.8.14/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -683,10 +683,10 @@ github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= -github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= @@ -809,8 +809,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -819,8 +819,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= -github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= +github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks= +github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4= github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30= github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk= github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws= @@ -828,8 +828,8 @@ github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= +github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -922,12 +922,12 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ= -github.com/viant/assertly v0.5.4/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY= -github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec= -github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE= -github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/viant/assertly v0.9.0 h1:uB3jO+qmWQcrSCHQRxA2kk88eXAdaklUUDxxCU5wBHQ= +github.com/viant/assertly v0.9.0/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/ptrie v0.3.1 h1:gIKbZBGET8ZcPkTZQMAUna9dHuLS8+bal+wl1fA1kHc= +github.com/viant/ptrie v0.3.1/go.mod h1:Y+mwwNCIUgFrCZcrG4/QChfi4ubvnNBsyrENBIgigu0= +github.com/viant/toolbox v0.34.5 h1:szWNPiGHjo8Dd4v2a59saEhG31DRL2Xf3aJ0ZtTSuqc= +github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -948,8 +948,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfG github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= -github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1 h1:AtNjus6Wm5pqK4pfYa9QQ83A2omHpzZuBwILB+ylfJ4= -github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1/go.mod h1:h7mpZZIAPq65QDZOO/qGVKyadsXzvv2ncs5fKuIho24= +github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0 h1:rWU5frxoj7C5Qpy4+7vs1aq/xEjIgpHK761pumHWCug= +github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0/go.mod h1:5Muw8UI+7corCqqx/OXAiboCF3jehxyev20B79Y1evk= github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo= github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -964,8 +964,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -981,8 +981,8 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= -go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= -go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= +go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= @@ -1018,8 +1018,8 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.3.1-0.20231011042131-892b665398ec h1:aB0WVMCyiVcqL1yMRLM4htiFlMvgdOml97GYnw9su5Q= -go.uber.org/mock v0.3.1-0.20231011042131-892b665398ec/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -1031,8 +1031,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gocloud.dev v0.37.0 h1:XF1rN6R0qZI/9DYjN16Uy0durAmSlf58DHOcb28GPro= gocloud.dev v0.37.0/go.mod h1:7/O4kqdInCNsc6LqgmuFnS0GRew4XNNYWpA44yQnwco= -gocloud.dev/pubsub/natspubsub v0.36.0 h1:QGBT53UN7VO7zttlGhAcLrwXjv0BKUbMET7hIlq5ehs= -gocloud.dev/pubsub/natspubsub v0.36.0/go.mod h1:OqWUmvFVNlPFF4YQ2PS7+/TnsEv/2loNfC1Uyp2plfA= +gocloud.dev/pubsub/natspubsub v0.37.0 h1:pnszLOYtyOkwB8oqiwYA11OcJpZ0MGJYBhnKl8c3tCs= +gocloud.dev/pubsub/natspubsub v0.37.0/go.mod h1:EN6ORFOviXxR/KALVQjTNh34EUcYQzMke6BCC6Czb8g= gocloud.dev/pubsub/rabbitpubsub v0.36.0 h1:f7KXLhk5HVZ81a2LKfTQY/HBDyXF0teQ7aMuJN+iK7I= gocloud.dev/pubsub/rabbitpubsub v0.36.0/go.mod h1:xCbgXeLHXrb2Yqs3WOTvmTvk6WeIfMMn3uq34uOk9yM= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= @@ -1246,10 +1246,10 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1261,8 +1261,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1364,8 +1364,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= -google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= +google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1412,8 +1412,8 @@ google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhl google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/k8s/charts/seaweedfs/templates/s3-ingress.yaml b/k8s/charts/seaweedfs/templates/s3-ingress.yaml index 7b279793b..08d58f662 100644 --- a/k8s/charts/seaweedfs/templates/s3-ingress.yaml +++ b/k8s/charts/seaweedfs/templates/s3-ingress.yaml @@ -10,9 +10,9 @@ kind: Ingress metadata: name: ingress-{{ template "seaweedfs.name" . }}-s3 namespace: {{ .Release.Namespace }} - {{- with .Values.s3.ingress.annotations }} + {{- if .Values.s3.ingress.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- tpl .Values.s3.ingress.annotations . | nindent 4 }} {{- end }} labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 9381f9046..676d0ca6f 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -293,9 +293,9 @@ volume: # same applies to "logs" - idx: "" + idx: {} - logs: "" + logs: {} # limit background compaction or copying speed in mega bytes per second compactionMBps: "50" @@ -746,7 +746,7 @@ s3: # host: false for "*" hostname host: "seaweedfs.cluster.local" # additional ingress annotations for the s3 endpoint - annotations: [] + annotations: "" tls: [] certificates: diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 60a5c538b..7ea2f0353 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -107,7 +107,7 @@ func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFi glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) return err } - err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0) + err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0) if err != nil { return err } @@ -123,7 +123,7 @@ func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunction return util.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset) } -func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) { +func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) { var shouldRetry bool var totalWritten int @@ -132,7 +132,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKe for _, urlString := range urlStrings { var localProcessed int var writeErr error - shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { + shouldRetry, err = util.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { if totalWritten > localProcessed { toBeSkipped := totalWritten - localProcessed if len(data) <= toBeSkipped { diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index cc3e58363..986dadb77 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u } } } - if count < limit && lastFileName <= prefix { + if count < limit && lastFileName < prefix { notPrefixed = notPrefixed[:0] lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry) diff --git a/weed/filer/stream.go b/weed/filer/stream.go index 2686fd833..23a853b9a 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -69,11 +69,17 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R type DoStreamContent func(writer io.Writer) error -func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) { - return PrepareStreamContentWithThrottler(masterClient, chunks, offset, size, 0) +func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) { + return PrepareStreamContentWithThrottler(masterClient, jwtFunc, chunks, offset, size, 0) } -func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) { +type VolumeServerJwtFunction func(fileId string) string + +func noJwtFunc(string) string { + return "" +} + +func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) { glog.V(4).Infof("prepare to stream content for chunks: %d", len(chunks)) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) @@ -119,7 +125,8 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc } urlStrings := fileId2Url[chunkView.FileId] start := time.Now() - err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize)) + jwt := jwtFunc(chunkView.FileId) + err := retriedStreamFetchChunkData(writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize)) offset += int64(chunkView.ViewSize) remaining -= int64(chunkView.ViewSize) stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds()) @@ -143,7 +150,7 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc } func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { - streamFn, err := PrepareStreamContent(masterClient, chunks, offset, size) + streamFn, err := PrepareStreamContent(masterClient, noJwtFunc, chunks, offset, size) if err != nil { return err } diff --git a/weed/ftpd/ftp_server.go b/weed/ftpd/ftp_server.go index d51f3624d..7334fa3c7 100644 --- a/weed/ftpd/ftp_server.go +++ b/weed/ftpd/ftp_server.go @@ -29,7 +29,7 @@ type SftpServer struct { var _ = ftpserver.MainDriver(&SftpServer{}) -// NewServer returns a new FTP server driver +// NewFtpServer returns a new FTP server driver func NewFtpServer(ftpListener net.Listener, option *FtpServerOption) (*SftpServer, error) { var err error server := &SftpServer{ diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index b6b870c3c..1b7a0146d 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "google.golang.org/grpc" "sync" @@ -193,6 +194,7 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest }) if lastError != nil { + stats.FilerHandlerCounter.WithLabelValues(stats.ErrorChunkAssign).Inc() continue } @@ -262,6 +264,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a WritableVolumeCount: so.VolumeGrowthCount, } if so.DataCenter != "" || so.Rack != "" || so.DataNode != "" { + ar.WritableVolumeCount = uint32(count) altRequest = &VolumeAssignRequest{ Count: uint64(count), Replication: so.Replication, diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 04548cc6f..0a156cfce 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -311,7 +311,7 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) { return signature, s3err.ErrNone } -// doesPolicySignatureMatch - Verify query headers with post policy +// doesPolicySignatureV4Match - Verify query headers with post policy // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // // returns ErrNone if the signature matches. diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index ccee8b885..6ff67b5bf 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -262,7 +262,7 @@ func getMD5HashBase64(data []byte) string { return base64.StdEncoding.EncodeToString(getMD5Sum(data)) } -// getSHA256Hash returns SHA-256 sum of given data. +// getSHA256Sum returns SHA-256 sum of given data. func getSHA256Sum(data []byte) []byte { hash := sha256.New() hash.Write(data) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 765a5679e..e9cd6a0c4 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1,11 +1,12 @@ package s3api import ( + "cmp" "encoding/hex" "encoding/xml" "fmt" - "github.com/google/uuid" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/stats" "golang.org/x/exp/slices" "math" "path/filepath" @@ -16,12 +17,19 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" + "github.com/google/uuid" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) +const ( + multipartExt = ".part" + multiPartMinSize = 5 * 1024 * 1024 +) + type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"` s3.CreateMultipartUploadOutput @@ -70,61 +78,129 @@ type CompleteMultipartUploadResult struct { func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("completeMultipartUpload input %v", input) - - completedParts := parts.Parts - slices.SortFunc(completedParts, func(a, b CompletedPart) int { - return a.PartNumber - b.PartNumber - }) + if len(parts.Parts) == 0 { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() + return nil, s3err.ErrNoSuchUpload + } + completedPartNumbers := []int{} + completedPartMap := make(map[int][]string) + for _, part := range parts.Parts { + if _, ok := completedPartMap[part.PartNumber]; !ok { + completedPartNumbers = append(completedPartNumbers, part.PartNumber) + } + completedPartMap[part.PartNumber] = append(completedPartMap[part.PartNumber], part.ETag) + } + sort.Ints(completedPartNumbers) uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId - entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) - if err != nil || len(entries) == 0 { + if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() + return nil, s3err.ErrNoSuchUpload + } + + if len(entries) == 0 { + entryName, dirName := s3a.getEntryNameAndDir(input) + if entry, _ := s3a.getEntry(dirName, entryName); entry != nil && entry.Extended != nil { + if uploadId, ok := entry.Extended[s3_constants.X_SeaweedFS_Header_Upload_Id]; ok && *input.UploadId == string(uploadId) { + return &CompleteMultipartUploadResult{ + CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{ + Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Bucket: input.Bucket, + ETag: aws.String("\"" + filer.ETagChunks(entry.GetChunks()) + "\""), + Key: objectKey(input.Key), + }, + }, s3err.ErrNone + } + } + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } - - // check whether completedParts is more than received parts - { - partNumbers := make(map[int]struct{}, len(entries)) - for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partNumberString := entry.Name[:len(entry.Name)-len(".part")] - partNumber, err := strconv.Atoi(partNumberString) - if err == nil { - partNumbers[partNumber] = struct{}{} + deleteEntries := []*filer_pb.Entry{} + partEntries := make(map[int][]*filer_pb.Entry, len(entries)) + entityTooSmall := false + for _, entry := range entries { + foundEntry := false + glog.V(4).Infof("completeMultipartUpload part entries %s", entry.Name) + if entry.IsDirectory || !strings.HasSuffix(entry.Name, multipartExt) { + continue + } + partNumber, err := parsePartNumber(entry.Name) + if err != nil { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartNumber).Inc() + glog.Errorf("completeMultipartUpload failed to pasre partNumber %s:%s", entry.Name, err) + continue + } + completedPartsByNumber, ok := completedPartMap[partNumber] + if !ok { + continue + } + for _, partETag := range completedPartsByNumber { + partETag = strings.Trim(partETag, `"`) + entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) + if partETag != "" && len(partETag) == 32 && entryETag != "" { + if entryETag != partETag { + glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedEtagMismatch).Inc() + continue } + } else { + glog.Warningf("invalid complete etag %s, partEtag %s", partETag, entryETag) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedEtagInvalid).Inc() + } + if len(entry.Chunks) == 0 { + glog.Warningf("completeMultipartUpload %s empty chunks", entry.Name) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartEmpty).Inc() + continue } + //there maybe multi same part, because of client retry + partEntries[partNumber] = append(partEntries[partNumber], entry) + foundEntry = true } - for _, part := range completedParts { - if _, found := partNumbers[part.PartNumber]; !found { - return nil, s3err.ErrInvalidPart + if foundEntry { + if len(completedPartNumbers) > 1 && partNumber != completedPartNumbers[len(completedPartNumbers)-1] && + entry.Attributes.FileSize < multiPartMinSize { + glog.Warningf("completeMultipartUpload %s part file size less 5mb", entry.Name) + entityTooSmall = true } + } else { + deleteEntries = append(deleteEntries, entry) } } - + if entityTooSmall { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompleteEntityTooSmall).Inc() + return nil, s3err.ErrEntityTooSmall + } mime := pentry.Attributes.Mime - var finalParts []*filer_pb.FileChunk var offset int64 - - for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partETag, found := findByPartNumber(entry.Name, completedParts) - if !found { + for _, partNumber := range completedPartNumbers { + partEntriesByNumber, ok := partEntries[partNumber] + if !ok { + glog.Errorf("part %d has no entry", partNumber) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartNotFound).Inc() + return nil, s3err.ErrInvalidPart + } + found := false + if len(partEntriesByNumber) > 1 { + slices.SortFunc(partEntriesByNumber, func(a, b *filer_pb.Entry) int { + return cmp.Compare(b.Chunks[0].ModifiedTsNs, a.Chunks[0].ModifiedTsNs) + }) + } + for _, entry := range partEntriesByNumber { + if found { + deleteEntries = append(deleteEntries, entry) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartEntryMismatch).Inc() continue } - entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) - if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { - glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) - return nil, s3err.ErrInvalidPart - } for _, chunk := range entry.GetChunks() { p := &filer_pb.FileChunk{ FileId: chunk.GetFileIdString(), @@ -137,28 +213,16 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa finalParts = append(finalParts, p) offset += int64(chunk.Size) } + found = true } } - entryName := filepath.Base(*input.Key) - dirName := filepath.ToSlash(filepath.Dir(*input.Key)) - if dirName == "." { - dirName = "" - } - if strings.HasPrefix(dirName, "/") { - dirName = dirName[1:] - } - dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName) - - // remove suffix '/' - if strings.HasSuffix(dirName, "/") { - dirName = dirName[:len(dirName)-1] - } - + entryName, dirName := s3a.getEntryNameAndDir(input) err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) { if entry.Extended == nil { entry.Extended = make(map[string][]byte) } + entry.Extended[s3_constants.X_SeaweedFS_Header_Upload_Id] = []byte(*input.UploadId) for k, v := range pentry.Extended { if k != "key" { entry.Extended[k] = v @@ -186,6 +250,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa }, } + for _, deleteEntry := range deleteEntries { + //delete unused part data + glog.Infof("completeMultipartUpload cleanup %s upload %s unused %s", *input.Bucket, *input.UploadId, deleteEntry.Name) + if err = s3a.rm(uploadDirectory, deleteEntry.Name, true, true); err != nil { + glog.Warningf("completeMultipartUpload cleanup %s upload %s unused %s : %v", *input.Bucket, *input.UploadId, deleteEntry.Name, err) + } + } if err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil { glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err) } @@ -193,29 +264,33 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } -func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) { - partNumber, formatErr := strconv.Atoi(fileName[:4]) - if formatErr != nil { - return +func (s3a *S3ApiServer) getEntryNameAndDir(input *s3.CompleteMultipartUploadInput) (string, string) { + entryName := filepath.Base(*input.Key) + dirName := filepath.ToSlash(filepath.Dir(*input.Key)) + if dirName == "." { + dirName = "" } - x := sort.Search(len(parts), func(i int) bool { - return parts[i].PartNumber >= partNumber - }) - if x >= len(parts) { - return + if strings.HasPrefix(dirName, "/") { + dirName = dirName[1:] } - if parts[x].PartNumber != partNumber { - return + dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName) + + // remove suffix '/' + if strings.HasSuffix(dirName, "/") { + dirName = dirName[:len(dirName)-1] } - y := 0 - for i, part := range parts[x:] { - if part.PartNumber == partNumber { - y = i - } else { - break - } + return entryName, dirName +} + +func parsePartNumber(fileName string) (int, error) { + var partNumberString string + index := strings.Index(fileName, "_") + if index != -1 { + partNumberString = fileName[:index] + } else { + partNumberString = fileName[:len(fileName)-len(multipartExt)] } - return parts[x+y].ETag, true + return strconv.Atoi(partNumberString) } func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { @@ -331,7 +406,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP StorageClass: aws.String("STANDARD"), } - entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts)) + entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d%s", *input.PartNumberMarker, multipartExt), false, uint32(*input.MaxParts)) if err != nil { glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err) return nil, s3err.ErrNoSuchUpload @@ -343,9 +418,8 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP output.IsTruncated = aws.Bool(!isLast) for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partNumberString := entry.Name[:len(entry.Name)-len(".part")] - partNumber, err := strconv.Atoi(partNumberString) + if strings.HasSuffix(entry.Name, multipartExt) && !entry.IsDirectory { + partNumber, err := parsePartNumber(entry.Name) if err != nil { glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err) continue diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go index e76d903b8..7f75a40de 100644 --- a/weed/s3api/filer_multipart_test.go +++ b/weed/s3api/filer_multipart_test.go @@ -50,88 +50,27 @@ func TestListPartsResult(t *testing.T) { } -func Test_findByPartNumber(t *testing.T) { - type args struct { - fileName string - parts []CompletedPart - } - - parts := []CompletedPart{ - { - ETag: "xxx", - PartNumber: 1, - }, - { - ETag: "lll", - PartNumber: 1, - }, - { - ETag: "yyy", - PartNumber: 3, - }, - { - ETag: "zzz", - PartNumber: 5, - }, - } - +func Test_parsePartNumber(t *testing.T) { tests := []struct { - name string - args args - wantEtag string - wantFound bool + name string + fileName string + partNum int }{ { "first", - args{ - "0001.part", - parts, - }, - "lll", - true, + "0001_uuid.part", + 1, }, { "second", - args{ - "0002.part", - parts, - }, - "", - false, - }, - { - "third", - args{ - "0003.part", - parts, - }, - "yyy", - true, - }, - { - "fourth", - args{ - "0004.part", - parts, - }, - "", - false, - }, - { - "fifth", - args{ - "0005.part", - parts, - }, - "zzz", - true, + "0002.part", + 2, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotEtag, gotFound := findByPartNumber(tt.args.fileName, tt.args.parts) - assert.Equalf(t, tt.wantEtag, gotEtag, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) - assert.Equalf(t, tt.wantFound, gotFound, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) + partNumber, _ := parsePartNumber(tt.fileName) + assert.Equalf(t, tt.partNum, partNumber, "parsePartNumber(%v)", tt.fileName) }) } } diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 30a878ccb..8e4a2f8c7 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -39,6 +39,7 @@ const ( AmzTagCount = "x-amz-tagging-count" X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" + X_SeaweedFS_Header_Upload_Id = "X-Seaweedfs-Upload-Id" // S3 ACL headers AmzCannedAcl = "X-Amz-Acl" diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index bb3939571..04e1e00a4 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -351,7 +351,7 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr } -// DeleteBucketMetricsConfiguration Delete Bucket Lifecycle +// DeleteBucketLifecycleHandler Delete Bucket Lifecycle // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 8dc33f213..8d13fe17e 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -2,16 +2,17 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "modernc.org/strutil" "net/http" "net/url" "strconv" "strings" "time" + "modernc.org/strutil" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -170,8 +171,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req rangeHeader := r.Header.Get("x-amz-copy-source-range") - dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", - s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(dstBucket), uploadID, partID) + dstUrl := s3a.genPartUploadUrl(dstBucket, uploadID, partID) srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlEscapeObject(srcObject)) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 187022079..6fecdcf2d 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/google/uuid" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -247,8 +248,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) - uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", - s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) + uploadUrl := s3a.genPartUploadUrl(bucket, uploadID, partID) if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) @@ -271,6 +271,11 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder) } +func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string { + return fmt.Sprintf("http://%s%s/%s/%04d_%s.part", + s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) +} + // Generate uploadID hash string from object func (s3a *S3ApiServer) generateUploadID(object string) string { if strings.HasPrefix(object, "/") { diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index f332da856..b00e4630d 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -374,7 +374,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } if cursor.maxKeys <= 0 { cursor.isTruncated = true - return + continue } entry := resp.Entry nextMarker = entry.Name diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 356761f30..9880afee0 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -91,6 +91,7 @@ type FilerServer struct { secret security.SigningKey filer *filer.Filer filerGuard *security.Guard + volumeGuard *security.Guard grpcDialOption grpc.DialOption // metrics read from the master @@ -113,6 +114,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + volumeSigningKey := v.GetString("jwt.signing.key") + v.SetDefault("jwt.signing.expires_after_seconds", 10) + volumeExpiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds") + + volumeReadSigningKey := v.GetString("jwt.signing.read.key") + v.SetDefault("jwt.signing.read.expires_after_seconds", 60) + volumeReadExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds") + v.SetDefault("cors.allowed_origins.values", "*") allowedOrigins := v.GetString("cors.allowed_origins.values") @@ -145,6 +154,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer.Cipher = option.Cipher // we do not support IP whitelist right now fs.filerGuard = security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) + fs.volumeGuard = security.NewGuard([]string{}, volumeSigningKey, volumeExpiresAfterSec, volumeReadSigningKey, volumeReadExpiresAfterSec) fs.checkWithMaster() diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go index db46f00b3..e04994569 100644 --- a/weed/server/filer_server_handlers_proxy.go +++ b/weed/server/filer_server_handlers_proxy.go @@ -2,6 +2,7 @@ package weed_server import ( "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/mem" "io" @@ -20,6 +21,26 @@ func init() { }} } +func (fs *FilerServer) maybeAddVolumeJwtAuthorization(r *http.Request, fileId string, isWrite bool) { + encodedJwt := fs.maybeGetVolumeJwtAuthorizationToken(fileId, isWrite) + + if encodedJwt == "" { + return + } + + r.Header.Set("Authorization", "BEARER "+string(encodedJwt)) +} + +func (fs *FilerServer) maybeGetVolumeJwtAuthorizationToken(fileId string, isWrite bool) string { + var encodedJwt security.EncodedJwt + if isWrite { + encodedJwt = security.GenJwtForVolumeServer(fs.volumeGuard.SigningKey, fs.volumeGuard.ExpiresAfterSec, fileId) + } else { + encodedJwt = security.GenJwtForVolumeServer(fs.volumeGuard.ReadSigningKey, fs.volumeGuard.ReadExpiresAfterSec, fileId) + } + return string(encodedJwt) +} + func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Request, fileId string) { urlStrings, err := fs.filer.MasterClient.GetLookupFileIdFunction()(fileId) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index c139060e4..83411051d 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -15,6 +15,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util/mem" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -261,7 +262,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } - streamFn, err := filer.PrepareStreamContentWithThrottler(fs.filer.MasterClient, chunks, offset, size, fs.option.DownloadMaxBytesPs) + streamFn, err := filer.PrepareStreamContentWithThrottler(fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken, chunks, offset, size, fs.option.DownloadMaxBytesPs) if err != nil { stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadStream).Inc() glog.Errorf("failed to prepare stream content %s: %v", r.URL, err) @@ -277,3 +278,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) }, nil }) } + +func (fs *FilerServer) maybeGetVolumeReadJwtAuthorizationToken(fileId string) string { + return string(security.GenJwtForVolumeServer(fs.volumeGuard.ReadSigningKey, fs.volumeGuard.ReadExpiresAfterSec, fileId)) +} diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 2aede2d50..5839a6a73 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -97,6 +97,9 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest continue } dn := dnList.Head() + if dn == nil { + continue + } var replicas []*master_pb.Location for _, r := range dnList.Rest() { replicas = append(replicas, &master_pb.Location{ diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index c5e059f21..9dc6351a4 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -149,7 +149,9 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.maybeAddJwtAuthorization(w, fid, true) dn := dnList.Head() - + if dn == nil { + continue + } writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) return } diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 9b6a64e6f..b724f16f9 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -315,8 +315,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id) if !takeAction { - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } @@ -349,8 +349,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co return err } - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } } diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index f61f68e08..83391f047 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -241,7 +241,13 @@ var ( Name: "request_total", Help: "Counter of s3 requests.", }, []string{"type", "code", "bucket"}) - + S3HandlerCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "handler_total", + Help: "Counter of s3 server handlers.", + }, []string{"type"}) S3RequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: Namespace, @@ -292,6 +298,7 @@ func init() { Gather.MustRegister(VolumeServerResourceGauge) Gather.MustRegister(S3RequestCounter) + Gather.MustRegister(S3HandlerCounter) Gather.MustRegister(S3RequestHistogram) Gather.MustRegister(S3TimeToFirstByteHistogram) } diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index f97796e3b..c0a6e99be 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -40,6 +40,17 @@ const ( ErrorReadInternal = "read.internal.error" ErrorWriteEntry = "write.entry.failed" RepeatErrorUploadContent = "upload.content.repeat.failed" + ErrorChunkAssign = "chunkAssign.failed" ErrorReadCache = "read.cache.failed" ErrorReadStream = "read.stream.failed" + + // s3 handler + ErrorCompletedNoSuchUpload = "errorCompletedNoSuchUpload" + ErrorCompleteEntityTooSmall = "errorCompleteEntityTooSmall" + ErrorCompletedPartEmpty = "errorCompletedPartEmpty" + ErrorCompletedPartNumber = "errorCompletedPartNumber" + ErrorCompletedPartNotFound = "errorCompletedPartNotFound" + ErrorCompletedEtagInvalid = "errorCompletedEtagInvalid" + ErrorCompletedEtagMismatch = "errorCompletedEtagMismatch" + ErrorCompletedPartEntryMismatch = "errorCompletedPartEntryMismatch" ) diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 050ded613..68ffbd7e7 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -81,13 +81,20 @@ func (df *DiskFile) Close() error { if df.File == nil { return nil } - if err := df.Sync(); err != nil { - return err + err := df.Sync() + var err1 error + if df.File != nil { + // always try to close + err1 = df.File.Close() } - if err := df.File.Close(); err != nil { + // assume closed + df.File = nil + if err != nil { return err } - df.File = nil + if err1 != nil { + return err1 + } return nil } diff --git a/weed/topology/volume_growth_test.go b/weed/topology/volume_growth_test.go index a3473c677..04c5e8aeb 100644 --- a/weed/topology/volume_growth_test.go +++ b/weed/topology/volume_growth_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" "testing" "github.com/seaweedfs/seaweedfs/weed/sequence" @@ -419,11 +420,13 @@ func TestPickForWrite(t *testing.T) { Rack: "", DataNode: "", } + v := util.GetViper() + v.Set("master.volume_growth.threshold", 0.9) for _, rpStr := range []string{"001", "010", "100"} { rp, _ := super_block.NewReplicaPlacementFromString(rpStr) vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType) volumeGrowOption.ReplicaPlacement = rp - for _, dc := range []string{"", "dc1", "dc2", "dc3"} { + for _, dc := range []string{"", "dc1", "dc2", "dc3", "dc0"} { volumeGrowOption.DataCenter = dc for _, r := range []string{""} { volumeGrowOption.Rack = r @@ -432,8 +435,13 @@ func TestPickForWrite(t *testing.T) { continue } volumeGrowOption.DataNode = dn - fileId, count, _, _, err := topo.PickForWrite(1, volumeGrowOption, vl) - if err != nil { + fileId, count, _, shouldGrow, err := topo.PickForWrite(1, volumeGrowOption, vl) + if dc == "dc0" { + if err == nil || count != 0 || !shouldGrow { + fmt.Println(dc, r, dn, "pick for write should be with error") + t.Fail() + } + } else if err != nil { fmt.Println(dc, r, dn, "pick for write error :", err) t.Fail() } else if count == 0 { @@ -442,6 +450,9 @@ func TestPickForWrite(t *testing.T) { } else if len(fileId) == 0 { fmt.Println(dc, r, dn, "pick for write file id is empty") t.Fail() + } else if shouldGrow { + fmt.Println(dc, r, dn, "pick for write error : not should grow") + t.Fail() } } } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 278978292..d04552d35 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -234,13 +234,18 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) { } func (vl *VolumeLayout) isAllWritable(vid needle.VolumeId) bool { - for _, dn := range vl.vid2location[vid].list { - if v, getError := dn.GetVolumesById(vid); getError == nil { - if v.ReadOnly { - return false + if location, ok := vl.vid2location[vid]; ok { + for _, dn := range location.list { + if v, getError := dn.GetVolumesById(vid); getError == nil { + if v.ReadOnly { + return false + } } } + } else { + return false } + return true } @@ -301,7 +306,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() { shouldGrow = true } - return vid, count, locationList, shouldGrow, nil + return vid, count, locationList.Copy(), shouldGrow, nil } return 0, 0, nil, shouldGrow, errors.New("Strangely vid " + vid.String() + " is on no machine!") } @@ -336,7 +341,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi return } } - return vid, count, locationList, shouldGrow, fmt.Errorf("No writable volumes in DataCenter:%v Rack:%v DataNode:%v", option.DataCenter, option.Rack, option.DataNode) + return vid, count, locationList, true, fmt.Errorf("No writable volumes in DataCenter:%v Rack:%v DataNode:%v", option.DataCenter, option.Rack, option.DataNode) } func (vl *VolumeLayout) HasGrowRequest() bool { diff --git a/weed/topology/volume_location_list.go b/weed/topology/volume_location_list.go index c26f77104..127ad67eb 100644 --- a/weed/topology/volume_location_list.go +++ b/weed/topology/volume_location_list.go @@ -28,6 +28,9 @@ func (dnll *VolumeLocationList) Copy() *VolumeLocationList { func (dnll *VolumeLocationList) Head() *DataNode { //mark first node as master volume + if dnll.Length() == 0 { + return nil + } return dnll.list[0] } diff --git a/weed/util/http_util.go b/weed/util/http_util.go index d1505f673..7b3ac4bc4 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -53,11 +53,15 @@ func Post(url string, values url.Values) ([]byte, error) { // github.com/seaweedfs/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go // may need increasing http.Client.Timeout func Get(url string) ([]byte, bool, error) { + return GetAuthenticated(url, "") +} +func GetAuthenticated(url, jwt string) ([]byte, bool, error) { request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, true, err } + maybeAddAuth(request, jwt) request.Header.Add("Accept-Encoding", "gzip") response, err := client.Do(request) @@ -101,11 +105,15 @@ func Head(url string) (http.Header, error) { return r.Header, nil } -func Delete(url string, jwt string) error { - req, err := http.NewRequest("DELETE", url, nil) +func maybeAddAuth(req *http.Request, jwt string) { if jwt != "" { req.Header.Set("Authorization", "BEARER "+string(jwt)) } +} + +func Delete(url string, jwt string) error { + req, err := http.NewRequest("DELETE", url, nil) + maybeAddAuth(req, jwt) if err != nil { return err } @@ -133,9 +141,7 @@ func Delete(url string, jwt string) error { func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error) { req, err := http.NewRequest("DELETE", url, nil) - if jwt != "" { - req.Header.Set("Authorization", "BEARER "+string(jwt)) - } + maybeAddAuth(req, jwt) if err != nil { return } @@ -193,9 +199,7 @@ func DownloadFile(fileUrl string, jwt string) (filename string, header http.Head return "", nil, nil, err } - if len(jwt) > 0 { - req.Header.Set("Authorization", "BEARER "+jwt) - } + maybeAddAuth(req, jwt) response, err := client.Do(req) if err != nil { @@ -229,7 +233,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC if cipherKey != nil { var n int - _, err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { + _, err := readEncryptedUrl(fileUrl, "", cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { n = copy(buf, data) }) return int64(n), err @@ -298,11 +302,16 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC } func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) { + return ReadUrlAsStreamAuthenticated(fileUrl, "", cipherKey, isContentGzipped, isFullChunk, offset, size, fn) +} + +func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) { if cipherKey != nil { - return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) + return readEncryptedUrl(fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) } req, err := http.NewRequest("GET", fileUrl, nil) + maybeAddAuth(req, jwt) if err != nil { return false, err } @@ -354,8 +363,8 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is } -func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) { - encryptedData, retryable, err := Get(fileUrl) +func readEncryptedUrl(fileUrl, jwt string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) { + encryptedData, retryable, err := GetAuthenticated(fileUrl, jwt) if err != nil { return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err) } @@ -392,9 +401,7 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt req.Header.Add("Accept-Encoding", "gzip") } - if len(jwt) > 0 { - req.Header.Set("Authorization", "BEARER "+jwt) - } + maybeAddAuth(req, jwt) r, err := client.Do(req) if err != nil { diff --git a/weed/util/skiplist/skiplist.go b/weed/util/skiplist/skiplist.go index 21eed4b43..befb0389c 100644 --- a/weed/util/skiplist/skiplist.go +++ b/weed/util/skiplist/skiplist.go @@ -26,7 +26,7 @@ type SkipList struct { // elementCount int } -// NewSeedEps returns a new empty, initialized Skiplist. +// NewSeed returns a new empty, initialized Skiplist. // Given a seed, a deterministic height/list behaviour can be achieved. // Eps is used to compare keys given by the ExtractKey() function on equality. func NewSeed(seed int64, listStore ListStore) *SkipList {