From df40908e578fdf0d41044a5d521c961c95b0ab55 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 29 Mar 2024 04:17:59 +0500 Subject: [PATCH 01/32] fix panic 5435 (#5436) --- weed/server/master_grpc_server_assign.go | 3 +++ weed/server/master_server_handlers.go | 4 +++- weed/topology/volume_layout.go | 2 +- weed/topology/volume_location_list.go | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 2aede2d50..5839a6a73 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -97,6 +97,9 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest continue } dn := dnList.Head() + if dn == nil { + continue + } var replicas []*master_pb.Location for _, r := range dnList.Rest() { replicas = append(replicas, &master_pb.Location{ diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index c5e059f21..9dc6351a4 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -149,7 +149,9 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.maybeAddJwtAuthorization(w, fid, true) dn := dnList.Head() - + if dn == nil { + continue + } writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) return } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 278978292..516d6a947 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -301,7 +301,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() { shouldGrow = true } - return vid, count, locationList, shouldGrow, nil + return vid, count, locationList.Copy(), shouldGrow, nil } return 0, 0, nil, shouldGrow, errors.New("Strangely vid " + vid.String() + " is on no machine!") } diff --git a/weed/topology/volume_location_list.go b/weed/topology/volume_location_list.go index c26f77104..127ad67eb 100644 --- a/weed/topology/volume_location_list.go +++ b/weed/topology/volume_location_list.go @@ -28,6 +28,9 @@ func (dnll *VolumeLocationList) Copy() *VolumeLocationList { func (dnll *VolumeLocationList) Head() *DataNode { //mark first node as master volume + if dnll.Length() == 0 { + return nil + } return dnll.list[0] } From 9c1e0f5811c95841b5628952ab09e6b8e219e6cb Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 29 Mar 2024 12:38:27 +0500 Subject: [PATCH 02/32] [master] grow volumes if no writable volumes in current dataCenter (#5434) * grow volumes if no writable volumes in current dataCenter https://github.com/seaweedfs/seaweedfs/issues/3886 * fix tests with volume grow * automatic volume grow one volume * add ErrorChunkAssign metrics --- weed/operation/assign_file_id.go | 3 +++ weed/stats/metrics_names.go | 1 + weed/topology/volume_growth_test.go | 17 ++++++++++++++--- weed/topology/volume_layout.go | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index b6b870c3c..1b7a0146d 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "google.golang.org/grpc" "sync" @@ -193,6 +194,7 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest }) if lastError != nil { + stats.FilerHandlerCounter.WithLabelValues(stats.ErrorChunkAssign).Inc() continue } @@ -262,6 +264,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a WritableVolumeCount: so.VolumeGrowthCount, } if so.DataCenter != "" || so.Rack != "" || so.DataNode != "" { + ar.WritableVolumeCount = uint32(count) altRequest = &VolumeAssignRequest{ Count: uint64(count), Replication: so.Replication, diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index f97796e3b..ffb0c76f1 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -40,6 +40,7 @@ const ( ErrorReadInternal = "read.internal.error" ErrorWriteEntry = "write.entry.failed" RepeatErrorUploadContent = "upload.content.repeat.failed" + ErrorChunkAssign = "chunkAssign.failed" ErrorReadCache = "read.cache.failed" ErrorReadStream = "read.stream.failed" ) diff --git a/weed/topology/volume_growth_test.go b/weed/topology/volume_growth_test.go index a3473c677..04c5e8aeb 100644 --- a/weed/topology/volume_growth_test.go +++ b/weed/topology/volume_growth_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" "testing" "github.com/seaweedfs/seaweedfs/weed/sequence" @@ -419,11 +420,13 @@ func TestPickForWrite(t *testing.T) { Rack: "", DataNode: "", } + v := util.GetViper() + v.Set("master.volume_growth.threshold", 0.9) for _, rpStr := range []string{"001", "010", "100"} { rp, _ := super_block.NewReplicaPlacementFromString(rpStr) vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType) volumeGrowOption.ReplicaPlacement = rp - for _, dc := range []string{"", "dc1", "dc2", "dc3"} { + for _, dc := range []string{"", "dc1", "dc2", "dc3", "dc0"} { volumeGrowOption.DataCenter = dc for _, r := range []string{""} { volumeGrowOption.Rack = r @@ -432,8 +435,13 @@ func TestPickForWrite(t *testing.T) { continue } volumeGrowOption.DataNode = dn - fileId, count, _, _, err := topo.PickForWrite(1, volumeGrowOption, vl) - if err != nil { + fileId, count, _, shouldGrow, err := topo.PickForWrite(1, volumeGrowOption, vl) + if dc == "dc0" { + if err == nil || count != 0 || !shouldGrow { + fmt.Println(dc, r, dn, "pick for write should be with error") + t.Fail() + } + } else if err != nil { fmt.Println(dc, r, dn, "pick for write error :", err) t.Fail() } else if count == 0 { @@ -442,6 +450,9 @@ func TestPickForWrite(t *testing.T) { } else if len(fileId) == 0 { fmt.Println(dc, r, dn, "pick for write file id is empty") t.Fail() + } else if shouldGrow { + fmt.Println(dc, r, dn, "pick for write error : not should grow") + t.Fail() } } } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 516d6a947..3c1c31aef 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -336,7 +336,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi return } } - return vid, count, locationList, shouldGrow, fmt.Errorf("No writable volumes in DataCenter:%v Rack:%v DataNode:%v", option.DataCenter, option.Rack, option.DataNode) + return vid, count, locationList, true, fmt.Errorf("No writable volumes in DataCenter:%v Rack:%v DataNode:%v", option.DataCenter, option.Rack, option.DataNode) } func (vl *VolumeLayout) HasGrowRequest() bool { From f9ec92dd830f03e12fae281684a514d0306cda75 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 29 Mar 2024 19:57:27 +0500 Subject: [PATCH 03/32] [tests] add action s3 tests (#5427) --- .github/workflows/s3tests.yml | 189 ++++++++++++++++++++++++++++++++++ docker/compose/s3tests.conf | 2 +- 2 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/s3tests.yml diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml new file mode 100644 index 000000000..dfd3d4acf --- /dev/null +++ b/.github/workflows/s3tests.yml @@ -0,0 +1,189 @@ +name: "Ceph S3 tests" + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.head_ref }}/s3tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: docker + +jobs: + s3tests: + name: Ceph S3 tests + runs-on: ubuntu-22.04 + container: + image: docker.io/kmlebedev/ceph-s3-tests:0.0.2 + timeout-minutes: 30 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + id: go + + - name: Run Ceph S3 tests + timeout-minutes: 15 + env: + S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf + shell: bash + run: | + cd /__w/seaweedfs/seaweedfs/weed + go install -buildvcs=false + set -x + nohup weed -v 0 server -filer -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + sleep 10 + cd /s3-tests + tox -- \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \ + s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \ + s3tests_boto3/functional/test_s3.py::test_multi_object_delete \ + s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \ + s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \ + s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \ + s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \ + s3tests_boto3/functional/test_s3.py::test_object_write_expires \ + s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \ + s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \ + s3tests_boto3/functional/test_s3.py::test_object_write_file \ + s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \ + s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \ + s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \ + s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \ + s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \ + s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \ + s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \ + s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \ + s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ + s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests_boto3/functional/test_s3.py::test_bucket_head \ + s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \ + s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \ + s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \ + s3tests_boto3/functional/test_s3.py::test_object_anon_put \ + s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \ + s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \ + s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \ + s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \ + s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \ + s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \ + s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \ + s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \ + s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \ + s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \ + s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \ + s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \ + s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \ + s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \ + s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \ + s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \ + s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \ + s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \ + s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ + s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \ + s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \ + s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf index 68d9ddeb7..f8d5b4930 100644 --- a/docker/compose/s3tests.conf +++ b/docker/compose/s3tests.conf @@ -2,7 +2,7 @@ ## this section is just used for host, port and bucket_prefix # host set for rgw in vstart.sh -host = s3 +host = 127.0.0.1 # port set for rgw in vstart.sh port = 8000 From 829c0da3816e798261437c8cf6ceba3ae26b91b5 Mon Sep 17 00:00:00 2001 From: Nikita Borzykh Date: Sun, 31 Mar 2024 00:47:00 +0300 Subject: [PATCH 04/32] fix: adjust condition in prefixFilterEntries prevent infinite loop (#5440) --- weed/filer/filerstore_wrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index cc3e58363..986dadb77 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u } } } - if count < limit && lastFileName <= prefix { + if count < limit && lastFileName < prefix { notPrefixed = notPrefixed[:0] lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry) From 2e5aa8d03b49643a447d8207f4c1a4d28b6ba722 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:27:33 -0700 Subject: [PATCH 05/32] chore(deps): bump actions/dependency-review-action from 4.2.4 to 4.2.5 (#5445) --- .github/workflows/depsreview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index ad69107d9..b21259656 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - name: 'Dependency Review' - uses: actions/dependency-review-action@733dd5d4a5203f238c33806593ec0f5fc5343d8c + uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b From ef06d6f074876a1255d5d04014bcda05198fb7ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:59:56 -0700 Subject: [PATCH 06/32] chore(deps): bump google.golang.org/api from 0.169.0 to 0.172.0 (#5448) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 77d1d0167..2c3011b67 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 // indirect - github.com/googleapis/gax-go/v2 v2.12.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gorilla/mux v1.8.1 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -118,7 +118,7 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.169.0 + google.golang.org/api v0.172.0 google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/grpc v1.62.1 @@ -331,7 +331,7 @@ require ( golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index fd7701849..110bfd958 100644 --- a/go.sum +++ b/go.sum @@ -486,8 +486,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= -github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -1364,8 +1364,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= -google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= +google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1412,8 +1412,8 @@ google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhl google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E= google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From d6149f87a3edc86377e80052579a1fbd73e4be07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:01:18 -0700 Subject: [PATCH 07/32] chore(deps): bump github.com/cenkalti/backoff/v4 from 4.2.1 to 4.3.0 (#5446) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2c3011b67..e2f2de6ff 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.50.36 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 - github.com/cenkalti/backoff/v4 v4.2.1 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect diff --git a/go.sum b/go.sum index 110bfd958..3314a510e 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= From d41c1c29568bca6277e5132272fbda3b2aa48af9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:01:36 -0700 Subject: [PATCH 08/32] chore(deps): bump github.com/viant/ptrie from 0.3.0 to 0.3.1 (#5447) --- go.mod | 7 +++---- go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index e2f2de6ff..fecbb2437 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/fclairamb/ftpserverlib v0.24.0 github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-errors/errors v1.4.2 // indirect + github.com/go-errors/errors v1.5.1 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.11.0 github.com/go-sql-driver/mysql v1.8.0 @@ -96,9 +96,8 @@ require ( github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 github.com/valyala/bytebufferpool v1.0.0 - github.com/viant/assertly v0.5.4 // indirect - github.com/viant/ptrie v0.3.0 - github.com/viant/toolbox v0.33.2 // indirect + github.com/viant/ptrie v0.3.1 + github.com/viant/toolbox v0.34.5 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/go.sum b/go.sum index 3314a510e..266409afd 100644 --- a/go.sum +++ b/go.sum @@ -330,8 +330,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -922,12 +922,12 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ= -github.com/viant/assertly v0.5.4/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY= -github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec= -github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE= -github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/viant/assertly v0.9.0 h1:uB3jO+qmWQcrSCHQRxA2kk88eXAdaklUUDxxCU5wBHQ= +github.com/viant/assertly v0.9.0/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/ptrie v0.3.1 h1:gIKbZBGET8ZcPkTZQMAUna9dHuLS8+bal+wl1fA1kHc= +github.com/viant/ptrie v0.3.1/go.mod h1:Y+mwwNCIUgFrCZcrG4/QChfi4ubvnNBsyrENBIgigu0= +github.com/viant/toolbox v0.34.5 h1:szWNPiGHjo8Dd4v2a59saEhG31DRL2Xf3aJ0ZtTSuqc= +github.com/viant/toolbox v0.34.5/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= From c18a4c691190205b09dc469cbbd0bd30a6493748 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:01:57 -0700 Subject: [PATCH 09/32] chore(deps): bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 (#5449) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fecbb2437..94f575753 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.11.0 - github.com/go-sql-driver/mysql v1.8.0 + github.com/go-sql-driver/mysql v1.8.1 github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect diff --git a/go.sum b/go.sum index 266409afd..01fbd46ae 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4 github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= -github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4= -github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= From fab5f66a95fde6377aa87e12927c3bda5b6788e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:02:26 -0700 Subject: [PATCH 10/32] chore(deps): bump github.com/linxGnu/grocksdb from 1.8.12 to 1.8.14 (#5450) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 94f575753..9a55690f2 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/klauspost/reedsolomon v1.12.1 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 - github.com/linxGnu/grocksdb v1.8.12 + github.com/linxGnu/grocksdb v1.8.14 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect diff --git a/go.sum b/go.sum index 01fbd46ae..1fc6ca27c 100644 --- a/go.sum +++ b/go.sum @@ -627,8 +627,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.8.12 h1:1/pCztQUOa3BX/1gR3jSZDoaKFpeHFvQ1XrqZpSvZVo= -github.com/linxGnu/grocksdb v1.8.12/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.14 h1:HTgyYalNwBSG/1qCQUIott44wU5b2Y9Kr3z7SK5OfGQ= +github.com/linxGnu/grocksdb v1.8.14/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= From 53fdda82f028d0b86a0e695c1947faa83848d230 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:55:11 -0700 Subject: [PATCH 11/32] chore(deps): bump github.com/aws/aws-sdk-go-v2 from 1.26.0 to 1.26.1 (#5451) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9a55690f2..8f86b5f71 100644 --- a/go.mod +++ b/go.mod @@ -139,7 +139,7 @@ require ( github.com/Jille/raft-grpc-transport v1.5.0 github.com/arangodb/go-driver v1.6.1 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.26.0 + github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.27.7 github.com/aws/aws-sdk-go-v2/credentials v1.17.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 @@ -204,7 +204,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect - github.com/aws/smithy-go v1.20.1 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bradenaw/juniper v0.15.2 // indirect github.com/buengese/sgzip v0.1.1 // indirect diff --git a/go.sum b/go.sum index 1fc6ca27c..28f792090 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aws/aws-sdk-go v1.50.36 h1:PjWXHwZPuTLMR1NIb8nEjLucZBMzmf84TLoLbD8BZqk= github.com/aws/aws-sdk-go v1.50.36/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= -github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4= @@ -176,8 +176,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg19 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90= github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk= github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8= -github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= -github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= From 2fc37db77ee3d87052b7a872b1301b7d18b139a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:55:28 -0700 Subject: [PATCH 12/32] chore(deps): bump go.etcd.io/etcd/client/pkg/v3 from 3.5.12 to 3.5.13 (#5452) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8f86b5f71..03b48afa6 100644 --- a/go.mod +++ b/go.mod @@ -159,7 +159,7 @@ require ( github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1 - go.etcd.io/etcd/client/pkg/v3 v3.5.12 + go.etcd.io/etcd/client/pkg/v3 v3.5.13 go.uber.org/atomic v1.11.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) diff --git a/go.sum b/go.sum index 28f792090..535a4f63a 100644 --- a/go.sum +++ b/go.sum @@ -981,8 +981,8 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= -go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= -go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= +go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= From 94008a84a492ed784dc90df48eada545d748f8a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:56:07 -0700 Subject: [PATCH 13/32] chore(deps): bump github.com/schollz/progressbar/v3 from 3.14.1 to 3.14.2 (#5453) --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 03b48afa6..4f19a0644 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.66.0 github.com/rdleal/intervalst v1.3.0 - github.com/schollz/progressbar/v3 v3.14.1 + github.com/schollz/progressbar/v3 v3.14.2 github.com/shirou/gopsutil/v3 v3.24.1 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 @@ -289,7 +289,7 @@ require ( github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect github.com/rfjakob/eme v1.1.2 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/go.sum b/go.sum index 535a4f63a..d6f44e08f 100644 --- a/go.sum +++ b/go.sum @@ -809,8 +809,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -819,8 +819,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= -github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= +github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks= +github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4= github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30= github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk= github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws= @@ -1246,8 +1246,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1261,8 +1261,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From df9031e19a96a97f12613d7b4c90eb19b265d311 Mon Sep 17 00:00:00 2001 From: Nikita Borzykh Date: Tue, 2 Apr 2024 01:55:16 +0300 Subject: [PATCH 14/32] fix: ensure complete gRPC stream consumption in doListFilerEntries to prevent context canceled errors. (#5441) --- weed/s3api/s3api_objects_list_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index f332da856..b00e4630d 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -374,7 +374,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } if cursor.maxKeys <= 0 { cursor.isTruncated = true - return + continue } entry := resp.Entry nextMarker = entry.Name From 2a88da4de757630a7d35779867f24c66c71ae7a9 Mon Sep 17 00:00:00 2001 From: Henco Appel Date: Tue, 2 Apr 2024 16:31:40 +0100 Subject: [PATCH 15/32] fix: always close volume file (#4530) (#5459) If sync fails then close is never called. We should always be calling close on the file. Co-authored-by: Hendrik Appel --- weed/storage/backend/disk_file.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 050ded613..68ffbd7e7 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -81,13 +81,20 @@ func (df *DiskFile) Close() error { if df.File == nil { return nil } - if err := df.Sync(); err != nil { - return err + err := df.Sync() + var err1 error + if df.File != nil { + // always try to close + err1 = df.File.Close() } - if err := df.File.Close(); err != nil { + // assume closed + df.File = nil + if err != nil { return err } - df.File = nil + if err1 != nil { + return err1 + } return nil } From d5d8b8e2aef278fc5ca61b0140841c3909c1a5e8 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:06:19 +0500 Subject: [PATCH 16/32] fix panic at isAllWritable (#5457) fix panic https://github.com/seaweedfs/seaweedfs/issues/5456 --- weed/topology/volume_layout.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 3c1c31aef..d04552d35 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -234,13 +234,18 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) { } func (vl *VolumeLayout) isAllWritable(vid needle.VolumeId) bool { - for _, dn := range vl.vid2location[vid].list { - if v, getError := dn.GetVolumesById(vid); getError == nil { - if v.ReadOnly { - return false + if location, ok := vl.vid2location[vid]; ok { + for _, dn := range location.list { + if v, getError := dn.GetVolumesById(vid); getError == nil { + if v.ReadOnly { + return false + } } } + } else { + return false } + return true } From b19c9847c6241cc76e087a4b930a776ef5f07d11 Mon Sep 17 00:00:00 2001 From: skycope Date: Wed, 3 Apr 2024 11:18:42 +0800 Subject: [PATCH 17/32] fix completed multiupload lost data (#5460) If there are putObjectPart requests with the same uploadId during completeMultiPart, it can result in data loss. putObjectPart requests might be due to timeout retries. Co-authored-by: Yang Wang --- weed/s3api/filer_multipart.go | 76 +++++++++++++------ weed/s3api/s3api_object_copy_handlers.go | 12 +-- weed/s3api/s3api_object_multipart_handlers.go | 9 ++- 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 765a5679e..ac9bffe21 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -4,9 +4,6 @@ import ( "encoding/hex" "encoding/xml" "fmt" - "github.com/google/uuid" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "golang.org/x/exp/slices" "math" "path/filepath" "sort" @@ -14,6 +11,10 @@ import ( "strings" "time" + "github.com/google/uuid" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "golang.org/x/exp/slices" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -90,40 +91,55 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return nil, s3err.ErrNoSuchUpload } - // check whether completedParts is more than received parts - { - partNumbers := make(map[int]struct{}, len(entries)) - for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partNumberString := entry.Name[:len(entry.Name)-len(".part")] - partNumber, err := strconv.Atoi(partNumberString) - if err == nil { - partNumbers[partNumber] = struct{}{} - } + partEntries := make(map[int][]*filer_pb.Entry, len(entries)) + for _, entry := range entries { + glog.V(4).Infof("completeMultipartUpload part entries %s", entry.Name) + if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { + var partNumberString string + index := strings.Index(entry.Name, "_") + if index != -1 { + partNumberString = entry.Name[:index] + } else { + partNumberString = entry.Name[:len(entry.Name)-len(".part")] } - } - for _, part := range completedParts { - if _, found := partNumbers[part.PartNumber]; !found { - return nil, s3err.ErrInvalidPart + partNumber, err := strconv.Atoi(partNumberString) + if err != nil { + glog.Errorf("completeMultipartUpload failed to pasre partNumber %s:%s", partNumberString, err) + continue } + //there maybe multi same part, because of client retry + partEntries[partNumber] = append(partEntries[partNumber], entry) } + } mime := pentry.Attributes.Mime var finalParts []*filer_pb.FileChunk var offset int64 + var deleteEntries []*filer_pb.Entry + for _, part := range completedParts { + entries := partEntries[part.PartNumber] + // check whether completedParts is more than received parts + if len(entries) == 0 { + glog.Errorf("part %d has no entry", part.PartNumber) + return nil, s3err.ErrInvalidPart + } - for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partETag, found := findByPartNumber(entry.Name, completedParts) - if !found { + found := false + for _, entry := range entries { + if found { + deleteEntries = append(deleteEntries, entry) continue } + + partETag := strings.Trim(part.ETag, `"`) entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) + glog.Warningf("complete etag %s, partEtag %s", partETag, entryETag) if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { - glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) - return nil, s3err.ErrInvalidPart + err = fmt.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) + deleteEntries = append(deleteEntries, entry) + continue } for _, chunk := range entry.GetChunks() { p := &filer_pb.FileChunk{ @@ -137,7 +153,14 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa finalParts = append(finalParts, p) offset += int64(chunk.Size) } + found = true + err = nil + } + if err != nil { + glog.Errorf("%s", err) + return nil, s3err.ErrInvalidPart } + } entryName := filepath.Base(*input.Key) @@ -186,6 +209,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa }, } + for _, deleteEntry := range deleteEntries { + //delete unused part data + glog.Infof("completeMultipartUpload cleanup %s upload %s unused %s", *input.Bucket, *input.UploadId, deleteEntry.Name) + if err = s3a.rm(uploadDirectory, deleteEntry.Name, true, true); err != nil { + glog.Warningf("completeMultipartUpload cleanup %s upload %s unused %s : %v", *input.Bucket, *input.UploadId, deleteEntry.Name, err) + } + } if err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil { glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err) } diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 8dc33f213..8d13fe17e 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -2,16 +2,17 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "modernc.org/strutil" "net/http" "net/url" "strconv" "strings" "time" + "modernc.org/strutil" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -170,8 +171,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req rangeHeader := r.Header.Get("x-amz-copy-source-range") - dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", - s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(dstBucket), uploadID, partID) + dstUrl := s3a.genPartUploadUrl(dstBucket, uploadID, partID) srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlEscapeObject(srcObject)) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 187022079..6fecdcf2d 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/google/uuid" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -247,8 +248,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) - uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", - s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) + uploadUrl := s3a.genPartUploadUrl(bucket, uploadID, partID) if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) @@ -271,6 +271,11 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder) } +func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string { + return fmt.Sprintf("http://%s%s/%s/%04d_%s.part", + s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) +} + // Generate uploadID hash string from object func (s3a *S3ApiServer) generateUploadID(object string) string { if strings.HasPrefix(object, "/") { From b4def8f0007f63dc6cb4a1be7109b4af2461bd68 Mon Sep 17 00:00:00 2001 From: mergwyn Date: Wed, 3 Apr 2024 16:17:41 +0100 Subject: [PATCH 18/32] Correct type for volume logs and idx values (#5462) Correct type for volume logs and idx values (#1) Empty values now correctly set to {} --- k8s/charts/seaweedfs/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 9381f9046..c51b2eb3e 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -293,9 +293,9 @@ volume: # same applies to "logs" - idx: "" + idx: {} - logs: "" + logs: {} # limit background compaction or copying speed in mega bytes per second compactionMBps: "50" From 881c9229720d9556cc8d5cf13ec8d9b7275b5b86 Mon Sep 17 00:00:00 2001 From: mergwyn Date: Wed, 3 Apr 2024 17:09:34 +0100 Subject: [PATCH 19/32] helm chart: make s3 ingress annotations consistent (#5463) Make s3 ingress annotations consistent --- k8s/charts/seaweedfs/templates/s3-ingress.yaml | 4 ++-- k8s/charts/seaweedfs/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/s3-ingress.yaml b/k8s/charts/seaweedfs/templates/s3-ingress.yaml index 7b279793b..08d58f662 100644 --- a/k8s/charts/seaweedfs/templates/s3-ingress.yaml +++ b/k8s/charts/seaweedfs/templates/s3-ingress.yaml @@ -10,9 +10,9 @@ kind: Ingress metadata: name: ingress-{{ template "seaweedfs.name" . }}-s3 namespace: {{ .Release.Namespace }} - {{- with .Values.s3.ingress.annotations }} + {{- if .Values.s3.ingress.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- tpl .Values.s3.ingress.annotations . | nindent 4 }} {{- end }} labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index c51b2eb3e..676d0ca6f 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -746,7 +746,7 @@ s3: # host: false for "*" hostname host: "seaweedfs.cluster.local" # additional ingress annotations for the s3 endpoint - annotations: [] + annotations: "" tls: [] certificates: From 7aa25c113736f9078b34d169b9bfcd4a677a65c4 Mon Sep 17 00:00:00 2001 From: Nikita Borzykh Date: Thu, 4 Apr 2024 19:16:16 +0300 Subject: [PATCH 20/32] fix: correctly form partNumber from new entry.Name format in "listObjectParts" (#5470) --- weed/s3api/filer_multipart.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index ac9bffe21..7dd68466e 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -374,7 +374,13 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP for _, entry := range entries { if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partNumberString := entry.Name[:len(entry.Name)-len(".part")] + var partNumberString string + index := strings.Index(entry.Name, "_") + if index != -1 { + partNumberString = entry.Name[:index] + } else { + partNumberString = entry.Name[:len(entry.Name)-len(".part")] + } partNumber, err := strconv.Atoi(partNumberString) if err != nil { glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err) From d42a04cceb51d06ae6ec0e80ca8675969018d98b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:56:39 +0500 Subject: [PATCH 21/32] [s3] fix s3 test_multipart_resend_first_finishes_last (#5471) * try fix s3 test https://github.com/seaweedfs/seaweedfs/pull/5466 * add error handler metrics * refactor * refactor multipartExt * delete bad entry parts --- weed/s3api/filer_multipart.go | 149 ++++++++++++++--------------- weed/s3api/filer_multipart_test.go | 81 ++-------------- weed/stats/metrics.go | 9 +- weed/stats/metrics_names.go | 9 ++ 4 files changed, 101 insertions(+), 147 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 7dd68466e..e8491fce7 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1,9 +1,12 @@ package s3api import ( + "cmp" "encoding/hex" "encoding/xml" "fmt" + "github.com/seaweedfs/seaweedfs/weed/stats" + "golang.org/x/exp/slices" "math" "path/filepath" "sort" @@ -11,18 +14,18 @@ import ( "strings" "time" - "github.com/google/uuid" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "golang.org/x/exp/slices" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" + "github.com/google/uuid" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) +const multipartExt = ".part" + type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"` s3.CreateMultipartUploadOutput @@ -71,74 +74,97 @@ type CompleteMultipartUploadResult struct { func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("completeMultipartUpload input %v", input) - - completedParts := parts.Parts - slices.SortFunc(completedParts, func(a, b CompletedPart) int { - return a.PartNumber - b.PartNumber - }) - + completedPartNumbers := []int{} + completedPartMap := make(map[int][]string) + for _, part := range parts.Parts { + if _, ok := completedPartMap[part.PartNumber]; !ok { + completedPartNumbers = append(completedPartNumbers, part.PartNumber) + } + completedPartMap[part.PartNumber] = append(completedPartMap[part.PartNumber], part.ETag) + } uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) if err != nil || len(entries) == 0 { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } + deleteEntries := []*filer_pb.Entry{} partEntries := make(map[int][]*filer_pb.Entry, len(entries)) for _, entry := range entries { + foundEntry := false glog.V(4).Infof("completeMultipartUpload part entries %s", entry.Name) - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - var partNumberString string - index := strings.Index(entry.Name, "_") - if index != -1 { - partNumberString = entry.Name[:index] + if entry.IsDirectory || !strings.HasSuffix(entry.Name, multipartExt) { + continue + } + partNumber, err := parsePartNumber(entry.Name) + if err != nil { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartNumber).Inc() + glog.Errorf("completeMultipartUpload failed to pasre partNumber %s:%s", entry.Name, err) + continue + } + completedPartsByNumber, ok := completedPartMap[partNumber] + if !ok { + continue + } + for _, partETag := range completedPartsByNumber { + partETag = strings.Trim(partETag, `"`) + entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) + if partETag != "" && len(partETag) == 32 && entryETag != "" { + if entryETag != partETag { + glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedEtagMismatch).Inc() + continue + } } else { - partNumberString = entry.Name[:len(entry.Name)-len(".part")] + glog.Warningf("invalid complete etag %s, partEtag %s", partETag, entryETag) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedEtagInvalid).Inc() } - partNumber, err := strconv.Atoi(partNumberString) - if err != nil { - glog.Errorf("completeMultipartUpload failed to pasre partNumber %s:%s", partNumberString, err) + if len(entry.Chunks) == 0 { + glog.Warningf("completeMultipartUpload %s empty chunks", entry.Name) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartEmpty).Inc() continue } //there maybe multi same part, because of client retry partEntries[partNumber] = append(partEntries[partNumber], entry) + foundEntry = true + } + if !foundEntry { + deleteEntries = append(deleteEntries, entry) } - } mime := pentry.Attributes.Mime var finalParts []*filer_pb.FileChunk var offset int64 - var deleteEntries []*filer_pb.Entry - for _, part := range completedParts { - entries := partEntries[part.PartNumber] - // check whether completedParts is more than received parts - if len(entries) == 0 { - glog.Errorf("part %d has no entry", part.PartNumber) + sort.Ints(completedPartNumbers) + for _, partNumber := range completedPartNumbers { + partEntriesByNumber, ok := partEntries[partNumber] + if !ok { + glog.Errorf("part %d has no entry", partNumber) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartNotFound).Inc() return nil, s3err.ErrInvalidPart } - found := false - for _, entry := range entries { + if len(partEntriesByNumber) > 1 { + slices.SortFunc(partEntriesByNumber, func(a, b *filer_pb.Entry) int { + return cmp.Compare(b.Chunks[0].ModifiedTsNs, a.Chunks[0].ModifiedTsNs) + }) + } + for _, entry := range partEntriesByNumber { if found { deleteEntries = append(deleteEntries, entry) - continue - } - - partETag := strings.Trim(part.ETag, `"`) - entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) - glog.Warningf("complete etag %s, partEtag %s", partETag, entryETag) - if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { - err = fmt.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) - deleteEntries = append(deleteEntries, entry) + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedPartEntryMismatch).Inc() continue } for _, chunk := range entry.GetChunks() { @@ -154,13 +180,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa offset += int64(chunk.Size) } found = true - err = nil } - if err != nil { - glog.Errorf("%s", err) - return nil, s3err.ErrInvalidPart - } - } entryName := filepath.Base(*input.Key) @@ -223,29 +243,15 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } -func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) { - partNumber, formatErr := strconv.Atoi(fileName[:4]) - if formatErr != nil { - return - } - x := sort.Search(len(parts), func(i int) bool { - return parts[i].PartNumber >= partNumber - }) - if x >= len(parts) { - return - } - if parts[x].PartNumber != partNumber { - return +func parsePartNumber(fileName string) (int, error) { + var partNumberString string + index := strings.Index(fileName, "_") + if index != -1 { + partNumberString = fileName[:index] + } else { + partNumberString = fileName[:len(fileName)-len(multipartExt)] } - y := 0 - for i, part := range parts[x:] { - if part.PartNumber == partNumber { - y = i - } else { - break - } - } - return parts[x+y].ETag, true + return strconv.Atoi(partNumberString) } func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { @@ -361,7 +367,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP StorageClass: aws.String("STANDARD"), } - entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts)) + entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d%s", *input.PartNumberMarker, multipartExt), false, uint32(*input.MaxParts)) if err != nil { glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err) return nil, s3err.ErrNoSuchUpload @@ -373,15 +379,8 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP output.IsTruncated = aws.Bool(!isLast) for _, entry := range entries { - if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - var partNumberString string - index := strings.Index(entry.Name, "_") - if index != -1 { - partNumberString = entry.Name[:index] - } else { - partNumberString = entry.Name[:len(entry.Name)-len(".part")] - } - partNumber, err := strconv.Atoi(partNumberString) + if strings.HasSuffix(entry.Name, multipartExt) && !entry.IsDirectory { + partNumber, err := parsePartNumber(entry.Name) if err != nil { glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err) continue diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go index e76d903b8..7f75a40de 100644 --- a/weed/s3api/filer_multipart_test.go +++ b/weed/s3api/filer_multipart_test.go @@ -50,88 +50,27 @@ func TestListPartsResult(t *testing.T) { } -func Test_findByPartNumber(t *testing.T) { - type args struct { - fileName string - parts []CompletedPart - } - - parts := []CompletedPart{ - { - ETag: "xxx", - PartNumber: 1, - }, - { - ETag: "lll", - PartNumber: 1, - }, - { - ETag: "yyy", - PartNumber: 3, - }, - { - ETag: "zzz", - PartNumber: 5, - }, - } - +func Test_parsePartNumber(t *testing.T) { tests := []struct { - name string - args args - wantEtag string - wantFound bool + name string + fileName string + partNum int }{ { "first", - args{ - "0001.part", - parts, - }, - "lll", - true, + "0001_uuid.part", + 1, }, { "second", - args{ - "0002.part", - parts, - }, - "", - false, - }, - { - "third", - args{ - "0003.part", - parts, - }, - "yyy", - true, - }, - { - "fourth", - args{ - "0004.part", - parts, - }, - "", - false, - }, - { - "fifth", - args{ - "0005.part", - parts, - }, - "zzz", - true, + "0002.part", + 2, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotEtag, gotFound := findByPartNumber(tt.args.fileName, tt.args.parts) - assert.Equalf(t, tt.wantEtag, gotEtag, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) - assert.Equalf(t, tt.wantFound, gotFound, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts) + partNumber, _ := parsePartNumber(tt.fileName) + assert.Equalf(t, tt.partNum, partNumber, "parsePartNumber(%v)", tt.fileName) }) } } diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index f61f68e08..83391f047 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -241,7 +241,13 @@ var ( Name: "request_total", Help: "Counter of s3 requests.", }, []string{"type", "code", "bucket"}) - + S3HandlerCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "handler_total", + Help: "Counter of s3 server handlers.", + }, []string{"type"}) S3RequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: Namespace, @@ -292,6 +298,7 @@ func init() { Gather.MustRegister(VolumeServerResourceGauge) Gather.MustRegister(S3RequestCounter) + Gather.MustRegister(S3HandlerCounter) Gather.MustRegister(S3RequestHistogram) Gather.MustRegister(S3TimeToFirstByteHistogram) } diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index ffb0c76f1..cfc0fbeb0 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -43,4 +43,13 @@ const ( ErrorChunkAssign = "chunkAssign.failed" ErrorReadCache = "read.cache.failed" ErrorReadStream = "read.stream.failed" + + // s3 handler + ErrorCompletedNoSuchUpload = "errorCompletedNoSuchUpload" + ErrorCompletedPartEmpty = "ErrorCompletedPartEmpty" + ErrorCompletedPartNumber = "ErrorCompletedPartNumber" + ErrorCompletedPartNotFound = "errorCompletedPartNotFound" + ErrorCompletedEtagInvalid = "errorCompletedEtagInvalid" + ErrorCompletedEtagMismatch = "errorCompletedEtagMismatch" + ErrorCompletedPartEntryMismatch = "errorCompletedPartEntryMismatch" ) From 35cba720a5e0c94d49c5d6ab8e621c0e17ed869a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:51:22 +0500 Subject: [PATCH 22/32] [s3] add s3 pass test_multipart_upload (#5474) add s3 pass test_multipart_upload --- .github/workflows/s3tests.yml | 1 + Makefile | 2 +- weed/s3api/filer_multipart.go | 57 ++++++++++++++++++++++--------- weed/s3api/s3_constants/header.go | 1 + 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index dfd3d4acf..f7dba0c39 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -166,6 +166,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \ s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \ diff --git a/Makefile b/Makefile index 269b6ec91..869de2f47 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone" server: install - weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324 + weed -v 0 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324 benchmark: install warp_install pkill weed || true diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index e8491fce7..b2da8193f 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/xml" "fmt" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/stats" "golang.org/x/exp/slices" "math" @@ -85,19 +86,36 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) - if err != nil || len(entries) == 0 { + if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } + if len(entries) == 0 { + entryName, dirName := s3a.getEntryNameAndDir(input) + if entry, _ := s3a.getEntry(dirName, entryName); entry != nil && entry.Extended != nil { + if uploadId, ok := entry.Extended[s3_constants.X_SeaweedFS_Header_Upload_Id]; ok && *input.UploadId == string(uploadId) { + return &CompleteMultipartUploadResult{ + CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{ + Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Bucket: input.Bucket, + ETag: aws.String("\"" + filer.ETagChunks(entry.GetChunks()) + "\""), + Key: objectKey(input.Key), + }, + }, s3err.ErrNone + } + } + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() + return nil, s3err.ErrNoSuchUpload + } + pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err) stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() return nil, s3err.ErrNoSuchUpload } - deleteEntries := []*filer_pb.Entry{} partEntries := make(map[int][]*filer_pb.Entry, len(entries)) for _, entry := range entries { @@ -183,25 +201,12 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa } } - entryName := filepath.Base(*input.Key) - dirName := filepath.ToSlash(filepath.Dir(*input.Key)) - if dirName == "." { - dirName = "" - } - if strings.HasPrefix(dirName, "/") { - dirName = dirName[1:] - } - dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName) - - // remove suffix '/' - if strings.HasSuffix(dirName, "/") { - dirName = dirName[:len(dirName)-1] - } - + entryName, dirName := s3a.getEntryNameAndDir(input) err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) { if entry.Extended == nil { entry.Extended = make(map[string][]byte) } + entry.Extended[s3_constants.X_SeaweedFS_Header_Upload_Id] = []byte(*input.UploadId) for k, v := range pentry.Extended { if k != "key" { entry.Extended[k] = v @@ -243,6 +248,24 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } +func (s3a *S3ApiServer) getEntryNameAndDir(input *s3.CompleteMultipartUploadInput) (string, string) { + entryName := filepath.Base(*input.Key) + dirName := filepath.ToSlash(filepath.Dir(*input.Key)) + if dirName == "." { + dirName = "" + } + if strings.HasPrefix(dirName, "/") { + dirName = dirName[1:] + } + dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName) + + // remove suffix '/' + if strings.HasSuffix(dirName, "/") { + dirName = dirName[:len(dirName)-1] + } + return entryName, dirName +} + func parsePartNumber(fileName string) (int, error) { var partNumberString string index := strings.Index(fileName, "_") diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 30a878ccb..8e4a2f8c7 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -39,6 +39,7 @@ const ( AmzTagCount = "x-amz-tagging-count" X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" + X_SeaweedFS_Header_Upload_Id = "X-Seaweedfs-Upload-Id" // S3 ACL headers AmzCannedAcl = "X-Amz-Acl" From 3e25ed1b11ce734bb5f2f9faddfe4b9ca16270c9 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:52:35 +0500 Subject: [PATCH 23/32] [s3] add s3 pass test_multipart_upload_size_too_small (#5475) * add s3 pass test_multipart_upload_size_too_small * refactor metric names * return ErrNoSuchUpload if empty parts * fix test --- .github/workflows/s3tests.yml | 2 +- weed/s3api/filer_multipart.go | 28 ++++++++++++++++++++++------ weed/stats/metrics_names.go | 5 +++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index f7dba0c39..fe6ce4a80 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -169,6 +169,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \ s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \ @@ -181,7 +182,6 @@ jobs: s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \ s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \ s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \ - s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \ s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \ s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \ s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index b2da8193f..e9cd6a0c4 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -25,7 +25,10 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) -const multipartExt = ".part" +const ( + multipartExt = ".part" + multiPartMinSize = 5 * 1024 * 1024 +) type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"` @@ -75,6 +78,10 @@ type CompleteMultipartUploadResult struct { func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("completeMultipartUpload input %v", input) + if len(parts.Parts) == 0 { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() + return nil, s3err.ErrNoSuchUpload + } completedPartNumbers := []int{} completedPartMap := make(map[int][]string) for _, part := range parts.Parts { @@ -83,8 +90,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa } completedPartMap[part.PartNumber] = append(completedPartMap[part.PartNumber], part.ETag) } - uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId + sort.Ints(completedPartNumbers) + uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) @@ -118,6 +126,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa } deleteEntries := []*filer_pb.Entry{} partEntries := make(map[int][]*filer_pb.Entry, len(entries)) + entityTooSmall := false for _, entry := range entries { foundEntry := false glog.V(4).Infof("completeMultipartUpload part entries %s", entry.Name) @@ -156,16 +165,23 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa partEntries[partNumber] = append(partEntries[partNumber], entry) foundEntry = true } - if !foundEntry { + if foundEntry { + if len(completedPartNumbers) > 1 && partNumber != completedPartNumbers[len(completedPartNumbers)-1] && + entry.Attributes.FileSize < multiPartMinSize { + glog.Warningf("completeMultipartUpload %s part file size less 5mb", entry.Name) + entityTooSmall = true + } + } else { deleteEntries = append(deleteEntries, entry) } } - + if entityTooSmall { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompleteEntityTooSmall).Inc() + return nil, s3err.ErrEntityTooSmall + } mime := pentry.Attributes.Mime - var finalParts []*filer_pb.FileChunk var offset int64 - sort.Ints(completedPartNumbers) for _, partNumber := range completedPartNumbers { partEntriesByNumber, ok := partEntries[partNumber] if !ok { diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index cfc0fbeb0..c0a6e99be 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -46,8 +46,9 @@ const ( // s3 handler ErrorCompletedNoSuchUpload = "errorCompletedNoSuchUpload" - ErrorCompletedPartEmpty = "ErrorCompletedPartEmpty" - ErrorCompletedPartNumber = "ErrorCompletedPartNumber" + ErrorCompleteEntityTooSmall = "errorCompleteEntityTooSmall" + ErrorCompletedPartEmpty = "errorCompletedPartEmpty" + ErrorCompletedPartNumber = "errorCompletedPartNumber" ErrorCompletedPartNotFound = "errorCompletedPartNotFound" ErrorCompletedEtagInvalid = "errorCompletedEtagInvalid" ErrorCompletedEtagMismatch = "errorCompletedEtagMismatch" From 9e07a87fcba5e6b0986cd2dad1fa3d644ff4edc7 Mon Sep 17 00:00:00 2001 From: clonefetch <166312854+clonefetch@users.noreply.github.com> Date: Mon, 8 Apr 2024 22:19:02 +0800 Subject: [PATCH 24/32] chore: fix function names in comment (#5478) --- weed/ftpd/ftp_server.go | 2 +- weed/s3api/auth_signature_v4.go | 2 +- weed/s3api/auto_signature_v4_test.go | 2 +- weed/s3api/s3api_bucket_handlers.go | 2 +- weed/util/skiplist/skiplist.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/ftpd/ftp_server.go b/weed/ftpd/ftp_server.go index d51f3624d..7334fa3c7 100644 --- a/weed/ftpd/ftp_server.go +++ b/weed/ftpd/ftp_server.go @@ -29,7 +29,7 @@ type SftpServer struct { var _ = ftpserver.MainDriver(&SftpServer{}) -// NewServer returns a new FTP server driver +// NewFtpServer returns a new FTP server driver func NewFtpServer(ftpListener net.Listener, option *FtpServerOption) (*SftpServer, error) { var err error server := &SftpServer{ diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 04548cc6f..0a156cfce 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -311,7 +311,7 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) { return signature, s3err.ErrNone } -// doesPolicySignatureMatch - Verify query headers with post policy +// doesPolicySignatureV4Match - Verify query headers with post policy // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // // returns ErrNone if the signature matches. diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index ccee8b885..6ff67b5bf 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -262,7 +262,7 @@ func getMD5HashBase64(data []byte) string { return base64.StdEncoding.EncodeToString(getMD5Sum(data)) } -// getSHA256Hash returns SHA-256 sum of given data. +// getSHA256Sum returns SHA-256 sum of given data. func getSHA256Sum(data []byte) []byte { hash := sha256.New() hash.Write(data) diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index bb3939571..04e1e00a4 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -351,7 +351,7 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr } -// DeleteBucketMetricsConfiguration Delete Bucket Lifecycle +// DeleteBucketLifecycleHandler Delete Bucket Lifecycle // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/util/skiplist/skiplist.go b/weed/util/skiplist/skiplist.go index 21eed4b43..befb0389c 100644 --- a/weed/util/skiplist/skiplist.go +++ b/weed/util/skiplist/skiplist.go @@ -26,7 +26,7 @@ type SkipList struct { // elementCount int } -// NewSeedEps returns a new empty, initialized Skiplist. +// NewSeed returns a new empty, initialized Skiplist. // Given a seed, a deterministic height/list behaviour can be achieved. // Eps is used to compare keys given by the ExtractKey() function on equality. func NewSeed(seed int64, listStore ListStore) *SkipList { From 5c8e6014baabe84cde25eea00ea75d0be5f0aa8f Mon Sep 17 00:00:00 2001 From: Henco Appel Date: Mon, 8 Apr 2024 15:27:00 +0100 Subject: [PATCH 25/32] fix: filer authenticate with with volume server (#5480) --- weed/filer/filechunk_manifest.go | 6 ++-- weed/filer/stream.go | 17 +++++++--- weed/server/filer_server.go | 10 ++++++ weed/server/filer_server_handlers_proxy.go | 21 ++++++++++++ weed/server/filer_server_handlers_read.go | 7 +++- weed/util/http_util.go | 37 +++++++++++++--------- 6 files changed, 74 insertions(+), 24 deletions(-) diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 60a5c538b..7ea2f0353 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -107,7 +107,7 @@ func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFi glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) return err } - err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0) + err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0) if err != nil { return err } @@ -123,7 +123,7 @@ func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunction return util.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset) } -func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) { +func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) { var shouldRetry bool var totalWritten int @@ -132,7 +132,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKe for _, urlString := range urlStrings { var localProcessed int var writeErr error - shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { + shouldRetry, err = util.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { if totalWritten > localProcessed { toBeSkipped := totalWritten - localProcessed if len(data) <= toBeSkipped { diff --git a/weed/filer/stream.go b/weed/filer/stream.go index 2686fd833..23a853b9a 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -69,11 +69,17 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R type DoStreamContent func(writer io.Writer) error -func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) { - return PrepareStreamContentWithThrottler(masterClient, chunks, offset, size, 0) +func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) { + return PrepareStreamContentWithThrottler(masterClient, jwtFunc, chunks, offset, size, 0) } -func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) { +type VolumeServerJwtFunction func(fileId string) string + +func noJwtFunc(string) string { + return "" +} + +func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) { glog.V(4).Infof("prepare to stream content for chunks: %d", len(chunks)) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) @@ -119,7 +125,8 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc } urlStrings := fileId2Url[chunkView.FileId] start := time.Now() - err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize)) + jwt := jwtFunc(chunkView.FileId) + err := retriedStreamFetchChunkData(writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize)) offset += int64(chunkView.ViewSize) remaining -= int64(chunkView.ViewSize) stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds()) @@ -143,7 +150,7 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc } func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { - streamFn, err := PrepareStreamContent(masterClient, chunks, offset, size) + streamFn, err := PrepareStreamContent(masterClient, noJwtFunc, chunks, offset, size) if err != nil { return err } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 356761f30..9880afee0 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -91,6 +91,7 @@ type FilerServer struct { secret security.SigningKey filer *filer.Filer filerGuard *security.Guard + volumeGuard *security.Guard grpcDialOption grpc.DialOption // metrics read from the master @@ -113,6 +114,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + volumeSigningKey := v.GetString("jwt.signing.key") + v.SetDefault("jwt.signing.expires_after_seconds", 10) + volumeExpiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds") + + volumeReadSigningKey := v.GetString("jwt.signing.read.key") + v.SetDefault("jwt.signing.read.expires_after_seconds", 60) + volumeReadExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds") + v.SetDefault("cors.allowed_origins.values", "*") allowedOrigins := v.GetString("cors.allowed_origins.values") @@ -145,6 +154,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer.Cipher = option.Cipher // we do not support IP whitelist right now fs.filerGuard = security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) + fs.volumeGuard = security.NewGuard([]string{}, volumeSigningKey, volumeExpiresAfterSec, volumeReadSigningKey, volumeReadExpiresAfterSec) fs.checkWithMaster() diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go index db46f00b3..e04994569 100644 --- a/weed/server/filer_server_handlers_proxy.go +++ b/weed/server/filer_server_handlers_proxy.go @@ -2,6 +2,7 @@ package weed_server import ( "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/mem" "io" @@ -20,6 +21,26 @@ func init() { }} } +func (fs *FilerServer) maybeAddVolumeJwtAuthorization(r *http.Request, fileId string, isWrite bool) { + encodedJwt := fs.maybeGetVolumeJwtAuthorizationToken(fileId, isWrite) + + if encodedJwt == "" { + return + } + + r.Header.Set("Authorization", "BEARER "+string(encodedJwt)) +} + +func (fs *FilerServer) maybeGetVolumeJwtAuthorizationToken(fileId string, isWrite bool) string { + var encodedJwt security.EncodedJwt + if isWrite { + encodedJwt = security.GenJwtForVolumeServer(fs.volumeGuard.SigningKey, fs.volumeGuard.ExpiresAfterSec, fileId) + } else { + encodedJwt = security.GenJwtForVolumeServer(fs.volumeGuard.ReadSigningKey, fs.volumeGuard.ReadExpiresAfterSec, fileId) + } + return string(encodedJwt) +} + func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Request, fileId string) { urlStrings, err := fs.filer.MasterClient.GetLookupFileIdFunction()(fileId) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index c139060e4..83411051d 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -15,6 +15,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util/mem" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -261,7 +262,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } - streamFn, err := filer.PrepareStreamContentWithThrottler(fs.filer.MasterClient, chunks, offset, size, fs.option.DownloadMaxBytesPs) + streamFn, err := filer.PrepareStreamContentWithThrottler(fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken, chunks, offset, size, fs.option.DownloadMaxBytesPs) if err != nil { stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadStream).Inc() glog.Errorf("failed to prepare stream content %s: %v", r.URL, err) @@ -277,3 +278,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) }, nil }) } + +func (fs *FilerServer) maybeGetVolumeReadJwtAuthorizationToken(fileId string) string { + return string(security.GenJwtForVolumeServer(fs.volumeGuard.ReadSigningKey, fs.volumeGuard.ReadExpiresAfterSec, fileId)) +} diff --git a/weed/util/http_util.go b/weed/util/http_util.go index d1505f673..7b3ac4bc4 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -53,11 +53,15 @@ func Post(url string, values url.Values) ([]byte, error) { // github.com/seaweedfs/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go // may need increasing http.Client.Timeout func Get(url string) ([]byte, bool, error) { + return GetAuthenticated(url, "") +} +func GetAuthenticated(url, jwt string) ([]byte, bool, error) { request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, true, err } + maybeAddAuth(request, jwt) request.Header.Add("Accept-Encoding", "gzip") response, err := client.Do(request) @@ -101,11 +105,15 @@ func Head(url string) (http.Header, error) { return r.Header, nil } -func Delete(url string, jwt string) error { - req, err := http.NewRequest("DELETE", url, nil) +func maybeAddAuth(req *http.Request, jwt string) { if jwt != "" { req.Header.Set("Authorization", "BEARER "+string(jwt)) } +} + +func Delete(url string, jwt string) error { + req, err := http.NewRequest("DELETE", url, nil) + maybeAddAuth(req, jwt) if err != nil { return err } @@ -133,9 +141,7 @@ func Delete(url string, jwt string) error { func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error) { req, err := http.NewRequest("DELETE", url, nil) - if jwt != "" { - req.Header.Set("Authorization", "BEARER "+string(jwt)) - } + maybeAddAuth(req, jwt) if err != nil { return } @@ -193,9 +199,7 @@ func DownloadFile(fileUrl string, jwt string) (filename string, header http.Head return "", nil, nil, err } - if len(jwt) > 0 { - req.Header.Set("Authorization", "BEARER "+jwt) - } + maybeAddAuth(req, jwt) response, err := client.Do(req) if err != nil { @@ -229,7 +233,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC if cipherKey != nil { var n int - _, err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { + _, err := readEncryptedUrl(fileUrl, "", cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { n = copy(buf, data) }) return int64(n), err @@ -298,11 +302,16 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC } func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) { + return ReadUrlAsStreamAuthenticated(fileUrl, "", cipherKey, isContentGzipped, isFullChunk, offset, size, fn) +} + +func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) { if cipherKey != nil { - return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) + return readEncryptedUrl(fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) } req, err := http.NewRequest("GET", fileUrl, nil) + maybeAddAuth(req, jwt) if err != nil { return false, err } @@ -354,8 +363,8 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is } -func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) { - encryptedData, retryable, err := Get(fileUrl) +func readEncryptedUrl(fileUrl, jwt string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) { + encryptedData, retryable, err := GetAuthenticated(fileUrl, jwt) if err != nil { return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err) } @@ -392,9 +401,7 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt req.Header.Add("Accept-Encoding", "gzip") } - if len(jwt) > 0 { - req.Header.Set("Authorization", "BEARER "+jwt) - } + maybeAddAuth(req, jwt) r, err := client.Do(req) if err != nil { From 67ead9b18f0e7f23f690c7f04e521be9e4d88270 Mon Sep 17 00:00:00 2001 From: "steve.wei" Date: Mon, 8 Apr 2024 22:30:04 +0800 Subject: [PATCH 26/32] fix(volume.fix.replication): adjust volume count, not free volume count (#5479) --- weed/shell/command_volume_fix_replication.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 9b6a64e6f..b724f16f9 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -315,8 +315,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id) if !takeAction { - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } @@ -349,8 +349,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co return err } - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } } From da42173d16bc0adbb33fec80973e5018fe9c98a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:16:15 -0700 Subject: [PATCH 27/32] chore(deps): bump docker/setup-buildx-action from 3.2.0 to 3.3.0 (#5486) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/2b51285047da1547ffb1b2203d8be4c0af6b1f20...d70bba72b1f3fd22344832f00baa16ece964efeb) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index df2f9fed8..2c298cf4e 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -36,7 +36,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 6f103576e..ad6cb0861 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index c63135f38..d9da49920 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index e3682f463..612898014 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 41330815e..912045355 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 901fe0bc4..7fb6d602c 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index bc499dc7c..6e8dc4ccb 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v1 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' From 40ef5d05a34a8b6b1644eff13ee8439d90c57bec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:16:25 -0700 Subject: [PATCH 28/32] chore(deps): bump golang.org/x/sys from 0.18.0 to 0.19.0 (#5485) Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.18.0 to 0.19.0. - [Commits](https://github.com/golang/sys/compare/v0.18.0...v0.19.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4f19a0644..bc17e4bdd 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,7 @@ require ( golang.org/x/image v0.15.0 golang.org/x/net v0.22.0 golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index d6f44e08f..872962d86 100644 --- a/go.sum +++ b/go.sum @@ -1248,8 +1248,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 576a4376f40268dc585267460f5347dd038fda42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:16:33 -0700 Subject: [PATCH 29/32] chore(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.56.1 to 3.63.0 (#5484) chore(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.56.1 to 3.63.0. - [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases) - [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md) - [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.56.1...v3.63.0) --- updated-dependencies: - dependency-name: github.com/ydb-platform/ydb-go-sdk/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index bc17e4bdd..0fa94ff56 100644 --- a/go.mod +++ b/go.mod @@ -158,7 +158,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.1 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 - github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1 + github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0 go.etcd.io/etcd/client/pkg/v3 v3.5.13 go.uber.org/atomic v1.11.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 diff --git a/go.sum b/go.sum index 872962d86..36b90fe6b 100644 --- a/go.sum +++ b/go.sum @@ -948,8 +948,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfG github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= -github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1 h1:AtNjus6Wm5pqK4pfYa9QQ83A2omHpzZuBwILB+ylfJ4= -github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1/go.mod h1:h7mpZZIAPq65QDZOO/qGVKyadsXzvv2ncs5fKuIho24= +github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0 h1:rWU5frxoj7C5Qpy4+7vs1aq/xEjIgpHK761pumHWCug= +github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0/go.mod h1:5Muw8UI+7corCqqx/OXAiboCF3jehxyev20B79Y1evk= github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo= github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -1018,8 +1018,8 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.3.1-0.20231011042131-892b665398ec h1:aB0WVMCyiVcqL1yMRLM4htiFlMvgdOml97GYnw9su5Q= -go.uber.org/mock v0.3.1-0.20231011042131-892b665398ec/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= From 7aef1b88c96760246cd369b93e5905c7a5053f9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:16:43 -0700 Subject: [PATCH 30/32] chore(deps): bump github.com/aws/aws-sdk-go-v2/config from 1.27.7 to 1.27.11 (#5483) chore(deps): bump github.com/aws/aws-sdk-go-v2/config Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.27.7 to 1.27.11. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.27.7...config/v1.27.11) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 20 ++++++++++---------- go.sum | 40 ++++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 0fa94ff56..15e89a8c6 100644 --- a/go.mod +++ b/go.mod @@ -140,8 +140,8 @@ require ( github.com/arangodb/go-driver v1.6.1 github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go-v2 v1.26.1 - github.com/aws/aws-sdk-go-v2/config v1.27.7 - github.com/aws/aws-sdk-go-v2/credentials v1.17.7 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 github.com/fluent/fluent-logger-golang v1.9.0 github.com/getsentry/sentry-go v0.27.0 @@ -190,20 +190,20 @@ require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect github.com/aws/smithy-go v1.20.2 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bradenaw/juniper v0.15.2 // indirect diff --git a/go.sum b/go.sum index 36b90fe6b..58cb7e157 100644 --- a/go.sum +++ b/go.sum @@ -142,26 +142,26 @@ github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+ github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= -github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4= -github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 h1:mDnFOE2sVkyphMWtTH+stv0eW3k0OTx94K63xpxHty4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3/go.mod h1:V8MuRVcCRt5h1S+Fwu8KbC7l/gBGo3yBAyUbJM2IJOk= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 h1:mbWNpfRUTT6bnacmvOTKXZjR/HycibdWzNpfbrbLDIs= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5/go.mod h1:FCOPWGjsshkkICJIn9hq9xr6dLKtyaWpuUojiN3W1/8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 h1:4t+QEX7BsXz98W8W1lNvMAG+NX8qHz2CjLBxQKku40g= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3/go.mod h1:oFcjjUq5Hm09N9rpxTdeMeLeQcxS7mIkBkL8qUKng+A= github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 h1:lW5xUzOPGAMY7HPuNF4FdyBwRc3UJ/e8KsapbesVeNU= @@ -170,12 +170,12 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMz github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2/go.mod h1:J3XhTE+VsY1jDsdDY+ACFAppZj/gpvygzC5JE0bTLbQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 370c6586f0aaf675e507e4f3c3d6cf7808d1f290 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:16:55 -0700 Subject: [PATCH 31/32] chore(deps): bump gocloud.dev/pubsub/natspubsub from 0.36.0 to 0.37.0 (#5482) Bumps [gocloud.dev/pubsub/natspubsub](https://github.com/google/go-cloud) from 0.36.0 to 0.37.0. - [Release notes](https://github.com/google/go-cloud/releases) - [Commits](https://github.com/google/go-cloud/compare/v0.36.0...v0.37.0) --- updated-dependencies: - dependency-name: gocloud.dev/pubsub/natspubsub dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 15e89a8c6..4e7dba978 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/reedsolomon v1.12.1 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 @@ -106,7 +106,7 @@ require ( go.mongodb.org/mongo-driver v1.14.0 go.opencensus.io v0.24.0 // indirect gocloud.dev v0.37.0 - gocloud.dev/pubsub/natspubsub v0.36.0 + gocloud.dev/pubsub/natspubsub v0.37.0 gocloud.dev/pubsub/rabbitpubsub v0.36.0 golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 @@ -266,8 +266,8 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/montanaflynn/stats v0.7.0 // indirect - github.com/nats-io/nats.go v1.31.0 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/nats.go v1.33.1 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/ncw/swift/v2 v2.0.2 // indirect github.com/onsi/gomega v1.27.10 // indirect diff --git a/go.sum b/go.sum index 58cb7e157..b01ad7868 100644 --- a/go.sum +++ b/go.sum @@ -595,8 +595,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -683,10 +683,10 @@ github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= -github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= @@ -1031,8 +1031,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gocloud.dev v0.37.0 h1:XF1rN6R0qZI/9DYjN16Uy0durAmSlf58DHOcb28GPro= gocloud.dev v0.37.0/go.mod h1:7/O4kqdInCNsc6LqgmuFnS0GRew4XNNYWpA44yQnwco= -gocloud.dev/pubsub/natspubsub v0.36.0 h1:QGBT53UN7VO7zttlGhAcLrwXjv0BKUbMET7hIlq5ehs= -gocloud.dev/pubsub/natspubsub v0.36.0/go.mod h1:OqWUmvFVNlPFF4YQ2PS7+/TnsEv/2loNfC1Uyp2plfA= +gocloud.dev/pubsub/natspubsub v0.37.0 h1:pnszLOYtyOkwB8oqiwYA11OcJpZ0MGJYBhnKl8c3tCs= +gocloud.dev/pubsub/natspubsub v0.37.0/go.mod h1:EN6ORFOviXxR/KALVQjTNh34EUcYQzMke6BCC6Czb8g= gocloud.dev/pubsub/rabbitpubsub v0.36.0 h1:f7KXLhk5HVZ81a2LKfTQY/HBDyXF0teQ7aMuJN+iK7I= gocloud.dev/pubsub/rabbitpubsub v0.36.0/go.mod h1:xCbgXeLHXrb2Yqs3WOTvmTvk6WeIfMMn3uq34uOk9yM= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= From f08f95ac800b788e42290e58eb6444e094acf97f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:17:05 -0700 Subject: [PATCH 32/32] chore(deps): bump github.com/shirou/gopsutil/v3 from 3.24.1 to 3.24.3 (#5481) Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.24.1 to 3.24.3. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/compare/v3.24.1...v3.24.3) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4e7dba978..5d75ab429 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/rclone/rclone v1.66.0 github.com/rdleal/intervalst v1.3.0 github.com/schollz/progressbar/v3 v3.14.2 - github.com/shirou/gopsutil/v3 v3.24.1 + github.com/shirou/gopsutil/v3 v3.24.3 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 github.com/ydb-platform/ydb-go-sdk/v3 v3.63.0 @@ -313,7 +313,7 @@ require ( github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect go.etcd.io/bbolt v1.3.8 // indirect diff --git a/go.sum b/go.sum index b01ad7868..180cb8b4f 100644 --- a/go.sum +++ b/go.sum @@ -828,8 +828,8 @@ github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= +github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -964,8 +964,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=