Merge branch 'refs/heads/master' into list_recursive_prefixed_entries

8 months ago · 0c6dfc05f0
27 changed files with 2221 additions and 1598 deletions
--- a/go.mod
+++ b/go.mod
@ -4,7 +4,7 @@ go 1.22.0

 require (
 	cloud.google.com/go v0.112.2 // indirect
-	cloud.google.com/go/pubsub v1.37.0
+	cloud.google.com/go/pubsub v1.38.0
 	cloud.google.com/go/storage v1.40.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
@ -106,7 +106,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.37.0
 	golang.org/x/crypto v0.23.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
-	golang.org/x/image v0.15.0
+	golang.org/x/image v0.16.0
 	golang.org/x/net v0.25.0
 	golang.org/x/oauth2 v0.19.0 // indirect
 	golang.org/x/sys v0.20.0
@ -114,7 +114,7 @@ require (
 	golang.org/x/tools v0.21.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/api v0.177.0
-	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect
 	google.golang.org/grpc v1.63.2
 	google.golang.org/protobuf v1.34.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
@ -134,7 +134,7 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.26.1
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/getsentry/sentry-go v0.27.0
@ -149,10 +149,10 @@ require (
 	github.com/rclone/rclone v1.66.0
 	github.com/rdleal/intervalst v1.4.0
 	github.com/schollz/progressbar/v3 v3.14.2
-	github.com/shirou/gopsutil/v3 v3.24.3
+	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3
 	go.etcd.io/etcd/client/pkg/v3 v3.5.13
 	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
@ -309,7 +309,7 @@ require (
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
 	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@ -329,7 +329,7 @@ require (
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
--- a/go.sum
+++ b/go.sum
@ -32,14 +32,14 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
 cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
-cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
-cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
+cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs=
+cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ=
-cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ=
+cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc=
+cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
@ -168,8 +168,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/g
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
 github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMzKtU4d7FlAMtDm50=
 github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk=
@ -844,8 +844,8 @@ github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQ
 github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ=
 github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE=
-github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg=
+github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU=
+github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@ -958,14 +958,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca h1:PliQWLwi2gTSOk7QyYQ9GfjvvikmibLWmaplKHy+kfo=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfGLkaDHuAEnWX63/8RBBiA4=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0 h1:JHnWtMHGttTRgIDPB5waC9WYcaEsL4x4LWfrs/zd6jQ=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0/go.mod h1:hJnZV1xFlpwNeH5Hy3N2TlLYl18OmDP5P1puEsRDNrY=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3 h1:XeY8gUh18MNJW0+TmPx8rl+vpnWwwiqgII27JdPmgJA=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3/go.mod h1:hGX4CijskNnUTRgLlqMvZdrBQc1ALZgAnKHytF5nmj4=
 github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo=
 github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ=
 github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
@ -991,8 +991,8 @@ github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
 github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
 github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8=
-go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M=
+go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI=
+go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI=
 go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
 go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
@ -1018,8 +1018,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
 go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
 go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
 go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
@ -1088,8 +1088,8 @@ golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
-golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
+golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
+golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1266,7 +1266,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -1427,10 +1427,10 @@ google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s=
-google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U=
-google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc=
-google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
+google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw=
+google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww=
+google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
--- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
+++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
--- a/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml
+++ b/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml
@ -12,6 +12,9 @@ metadata:
    app.kubernetes.io/managed-by: {{ .Release.Service }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/component: filer
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
  endpoints:
    - interval: 30s
--- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml
@ -62,6 +62,9 @@ spec:
      initContainers:
        {{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
      {{- end }}
+      {{- if .Values.filer.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
      containers:
        - name: seaweedfs
          image: {{ template "filer.image" . }}
@ -84,11 +87,13 @@ spec:
                secretKeyRef:
                  name: secret-seaweedfs-db
                  key: user
+                  optional: true
            - name: WEED_MYSQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: secret-seaweedfs-db
                  key: password
+                  optional: true
            - name: SEAWEEDFS_FULLNAME
              value: "{{ template "seaweedfs.name" . }}"
            {{- if .Values.filer.extraEnvironmentVars }}
@ -262,6 +267,9 @@ spec:
          resources:
            {{ tpl .Values.filer.resources . | nindent 12 | trim }}
          {{- end }}
+          {{- if .Values.filer.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
        {{- if .Values.filer.sidecars }}
        {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
        {{- end }}
--- a/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml
+++ b/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml
@ -12,6 +12,9 @@ metadata:
    app.kubernetes.io/managed-by: {{ .Release.Service }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/component: master
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
  endpoints:
    - interval: 30s
--- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml
@ -61,6 +61,9 @@ spec:
      initContainers:
        {{ tpl .Values.master.initContainers . | nindent 8 | trim }}
      {{- end }}
+      {{- if .Values.master.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.master.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
      containers:
        - name: seaweedfs
          image: {{ template "master.image" . }}
@ -222,6 +225,9 @@ spec:
          resources:
            {{ tpl .Values.master.resources . | nindent 12 | trim }}
          {{- end }}
+          {{- if .Values.master.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
        {{- if .Values.master.sidecars }}
        {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
        {{- end }}
--- a/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml
+++ b/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml
@ -22,6 +22,9 @@ spec:
        app.kubernetes.io/instance: {{ .Release.Name | quote }}
    spec:
      restartPolicy: Never
+      {{- if .Values.filer.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
      containers:
      - name: post-install-job
        image: {{ template "master.image" . }}
@ -80,6 +83,9 @@ spec:
          {{- end }}
          - containerPort: {{ .Values.master.grpcPort }}
            #name: swfs-master-grpc
+        {{- if .Values.filer.containerSecurityContext.enabled }}
+        securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+        {{- end }}
    {{- if .Values.filer.s3.enableAuth }}
      volumes:
        - name: config-users
--- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml
+++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml
@ -50,6 +50,9 @@ spec:
      initContainers:
        {{ tpl .Values.s3.initContainers . | nindent 8 | trim }}
      {{- end }}
+      {{- if .Values.s3.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.s3.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
      containers:
        - name: seaweedfs
          image: {{ template "s3.image" . }}
@ -199,6 +202,9 @@ spec:
          resources:
            {{ tpl .Values.s3.resources . | nindent 12 | trim }}
          {{- end }}
+          {{- if .Values.s3.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
      {{- if .Values.s3.sidecars }}
      {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
      {{- end }}
--- a/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml
+++ b/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml
@ -12,6 +12,9 @@ metadata:
    app.kubernetes.io/managed-by: {{ .Release.Service }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/component: s3
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
  endpoints:
    - interval: 30s
--- a/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml
+++ b/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml
@ -12,6 +12,9 @@ metadata:
    app.kubernetes.io/managed-by: {{ .Release.Service }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/component: volume
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
  endpoints:
    - interval: 30s
--- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
@ -72,6 +72,9 @@ spec:
        {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
        {{- end }}
      {{- end }}
+      {{- if .Values.volume.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
      containers:
        - name: seaweedfs
          image: {{ template "volume.image" . }}
@ -237,6 +240,9 @@ spec:
          resources:
            {{ tpl .Values.volume.resources . | nindent 12 | trim }}
          {{- end }}
+          {{- if .Values.volume.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
      {{- if .Values.volume.sidecars }}
      {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
      {{- end }}
--- a/k8s/charts/seaweedfs/values.yaml
+++ b/k8s/charts/seaweedfs/values.yaml
@ -25,6 +25,7 @@ global:
    enabled: false
    gatewayHost: null
    gatewayPort: null
+    additionalLabels: {}
  # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
  enableReplication: false
  #  replication type is XYZ:
@ -166,6 +167,25 @@ master:
  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
  serviceAccountName: ""

+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
  ingress:
    enabled: false
    className: "nginx"
@ -377,6 +397,25 @@ volume:

  extraEnvironmentVars:

+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
  # used to configure livenessProbe on volume-server containers
  #
  livenessProbe:
@ -535,6 +574,25 @@ filer:
  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
  serviceAccountName: ""

+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
  ingress:
    enabled: false
    className: "nginx"
@ -605,7 +663,7 @@ filer:
    timeoutSeconds: 10

  # secret env variables
-  secretExtraEnvironmentVars: []
+  secretExtraEnvironmentVars: {}
      # WEED_POSTGRES_USERNAME:
      #   secretKeyRef:
      #     name: postgres-credentials
@ -706,6 +764,25 @@ s3:
  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
  serviceAccountName: ""

+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
  logs:
    type: "hostPath"
    size: ""
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@ -32,12 +32,15 @@ var cmdFix = &Command{
 var (
 	fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
 	fixVolumeId         = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
+	fixIncludeDeleted   = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
 	fixIgnoreError      = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
 )

 type VolumeFileScanner4Fix struct {
-	version needle.Version
-	nm      *needle_map.MemDb
+	version        needle.Version
+	nm             *needle_map.MemDb
+	nmDeleted      *needle_map.MemDb
+	includeDeleted bool
 }

 func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
@ -50,13 +53,20 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 }

 func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
-	glog.V(2).Infof("key %d offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
+	glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
 	if n.Size.IsValid() {
-		pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
-		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+		if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
+			return fmt.Errorf("saved %d with error %v", n.Size, pe)
+		}
 	} else {
-		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id)
+		if scanner.includeDeleted {
+			if pe := scanner.nmDeleted.Set(n.Id, types.ToOffset(offset), types.TombstoneFileSize); pe != nil {
+				return fmt.Errorf("saved deleted %d with error %v", n.Size, pe)
+			}
+		} else {
+			glog.V(2).Infof("skipping deleted file ...")
+			return scanner.nm.Delete(n.Id)
+		}
 	}
 	return nil
 }
@ -109,21 +119,45 @@ func runFix(cmd *Command, args []string) bool {
 			if *fixVolumeId != 0 && *fixVolumeId != volumeId {
 				continue
 			}
-			doFixOneVolume(basePath, baseFileName, collection, volumeId)
+			doFixOneVolume(basePath, baseFileName, collection, volumeId, *fixIncludeDeleted)
 		}
 	}
 	return true
 }

-func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) {
+func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
+	idxFile, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return
+	}
+	defer func() {
+		idxFile.Close()
+	}()
+
+	return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
+		_, err := idxFile.Write(value.ToBytes())
+		if scaner.includeDeleted && err == nil {
+			if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
+				_, err = idxFile.Write(deleted.ToBytes())
+			}
+		}
+		return err
+	})
+}
+
+func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
 	indexFileName := path.Join(basepath, baseFileName+".idx")

 	nm := needle_map.NewMemDb()
+	nmDeleted := needle_map.NewMemDb()
 	defer nm.Close()
+	defer nmDeleted.Close()

 	vid := needle.VolumeId(volumeId)
 	scanner := &VolumeFileScanner4Fix{
-		nm: nm,
+		nm:             nm,
+		nmDeleted:      nmDeleted,
+		includeDeleted: fixIncludeDeleted,
 	}

 	if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
@ -135,12 +169,12 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
 		}
 	}

-	if err := nm.SaveToIdx(indexFileName); err != nil {
-		os.Remove(indexFileName)
+	if err := SaveToIdx(scanner, indexFileName); err != nil {
 		err := fmt.Errorf("save to .idx File: %v", err)
 		if *fixIgnoreError {
 			glog.Error(err)
 		} else {
+			os.Remove(indexFileName)
 			glog.Fatal(err)
 		}
 	}
--- a/weed/command/scaffold/filer.toml
+++ b/weed/command/scaffold/filer.toml
@ -280,6 +280,13 @@ tls_client_key_file=""
 [mongodb]
 enabled = false
 uri = "mongodb://localhost:27017"
+username = ""
+password = ""
+ssl = false
+ssl_ca_file = ""
+ssl_cert_file = ""
+ssl_key_file = "
+insecure_skip_verify = false
 option_pool_size = 0
 database = "seaweedfs"

--- a/weed/filer/mongodb/mongodb_store.go
+++ b/weed/filer/mongodb/mongodb_store.go
@ -2,7 +2,12 @@ package mongodb

 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"os"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@ -10,7 +15,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"time"
 )

 func init() {
@ -37,17 +41,44 @@ func (store *MongodbStore) Initialize(configuration util.Configuration, prefix s
 	store.database = configuration.GetString(prefix + "database")
 	store.collectionName = "filemeta"
 	poolSize := configuration.GetInt(prefix + "option_pool_size")
-	return store.connection(configuration.GetString(prefix+"uri"), uint64(poolSize))
+	uri := configuration.GetString(prefix + "uri")
+	ssl := configuration.GetBool(prefix + "ssl")
+	sslCAFile := configuration.GetString(prefix + "ssl_ca_file")
+	sslCertFile := configuration.GetString(prefix + "ssl_cert_file")
+	sslKeyFile := configuration.GetString(prefix + "ssl_key_file")
+	username := configuration.GetString(prefix + "username")
+	password := configuration.GetString(prefix + "password")
+	insecure_skip_verify := configuration.GetBool(prefix + "insecure_skip_verify")
+
+	return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password, insecure_skip_verify)
 }

-func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
-	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string, insecure bool) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
 	opts := options.Client().ApplyURI(uri)

 	if poolSize > 0 {
 		opts.SetMaxPoolSize(poolSize)
 	}

+	if ssl {
+		tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile, insecure)
+		if err != nil {
+			return err
+		}
+		opts.SetTLSConfig(tlsConfig)
+	}
+
+	if username != "" && password != "" {
+		creds := options.Credential{
+			Username: username,
+			Password: password,
+		}
+		opts.SetAuth(creds)
+	}
+
 	client, err := mongo.Connect(ctx, opts)
 	if err != nil {
 		return err
@ -55,10 +86,36 @@ func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {

 	c := client.Database(store.database).Collection(store.collectionName)
 	err = store.indexUnique(c)
+
 	store.connect = client
 	return err
 }

+func configureTLS(caFile, certFile, keyFile string, insecure bool) (*tls.Config, error) {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not load client key pair: %s", err)
+	}
+
+	caCert, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not read CA certificate: %s", err)
+	}
+
+	caCertPool := x509.NewCertPool()
+	if !caCertPool.AppendCertsFromPEM(caCert) {
+		return nil, fmt.Errorf("failed to append CA certificate")
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates:       []tls.Certificate{cert},
+		RootCAs:            caCertPool,
+		InsecureSkipVerify: insecure,
+	}
+
+	return tlsConfig, nil
+}
+
 func (store *MongodbStore) createIndex(c *mongo.Collection, index mongo.IndexModel, opts *options.CreateIndexesOptions) error {
 	_, err := c.Indexes().CreateOne(context.Background(), index, opts)
 	return err
@ -93,13 +150,10 @@ func (store *MongodbStore) RollbackTransaction(ctx context.Context) error {
 }

 func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
-
 	return store.UpdateEntry(ctx, entry)
-
 }

 func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
-
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
 	if err != nil {
@ -126,7 +180,6 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry)
 }

 func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
-
 	dir, name := fullpath.DirAndName()
 	var data Model

@ -154,7 +207,6 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath
 }

 func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
-
 	dir, name := fullpath.DirAndName()

 	where := bson.M{"directory": dir, "name": name}
@ -167,7 +219,6 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
 }

 func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
-
 	where := bson.M{"directory": fullpath}
 	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)
 	if err != nil {
@ -186,7 +237,6 @@ func (store *MongodbStore) ListRecursivePrefixedEntries(ctx context.Context, dir
 }

 func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-
 	var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}}
 	if includeStartFile {
 		where["name"] = bson.M{
--- a/weed/iamapi/iamapi_management_handlers.go
+++ b/weed/iamapi/iamapi_management_handlers.go
@ -33,6 +33,7 @@ const (
 	StatementActionReadAcp  = "GetBucketAcl"
 	StatementActionList     = "List*"
 	StatementActionTagging  = "Tagging*"
+	StatementActionDelete   = "DeleteBucket*"
 )

 var (
@ -58,6 +59,8 @@ func MapToStatementAction(action string) string {
 		return s3_constants.ACTION_LIST
 	case StatementActionTagging:
 		return s3_constants.ACTION_TAGGING
+	case StatementActionDelete:
+		return s3_constants.ACTION_DELETE_BUCKET
 	default:
 		return ""
 	}
@ -79,6 +82,8 @@ func MapToIdentitiesAction(action string) string {
 		return StatementActionList
 	case s3_constants.ACTION_TAGGING:
 		return StatementActionTagging
+	case s3_constants.ACTION_DELETE_BUCKET:
+		return StatementActionDelete
 	default:
 		return ""
 	}
--- a/weed/replication/sink/s3sink/s3_sink.go
+++ b/weed/replication/sink/s3sink/s3_sink.go
@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"strconv"
 	"strings"
+	"encoding/base64"

 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@ -202,7 +203,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
 		Tagging: aws.String(tags),
 	}
 	if len(entry.Attributes.Md5) > 0 {
-		uploadInput.ContentMD5 = aws.String(fmt.Sprintf("%x", entry.Attributes.Md5))
+		uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
 	}
 	_, err = uploader.Upload(&uploadInput)

--- a/weed/s3api/auth_credentials.go
+++ b/weed/s3api/auth_credentials.go
@ -317,6 +317,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 		}

 		identity, errCode := iam.authRequest(r, action)
+		glog.V(3).Infof("auth error: %v", errCode)
 		if errCode == s3err.ErrNone {
 			if identity != nil && identity.Name != "" {
 				r.Header.Set(s3_constants.AmzIdentityId, identity.Name)
@ -453,6 +454,7 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
 		}
 	}
 	if bucket == "" {
+		glog.V(3).Infof("identity %s is not allowed to perform action %s on %s -- bucket is empty", identity.Name, action, bucket+objectKey)
 		return false
 	}
 	target := string(action) + ":" + bucket + objectKey
@ -477,6 +479,8 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string)
 			}
 		}
 	}
+	//log error
+	glog.V(3).Infof("identity %s is not allowed to perform action %s on %s", identity.Name, action, bucket+objectKey)
 	return false
 }

--- a/weed/s3api/auth_credentials_test.go
+++ b/weed/s3api/auth_credentials_test.go
@ -1,11 +1,12 @@
 package s3api

 import (
-	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
-	"github.com/stretchr/testify/assert"
 	"reflect"
 	"testing"

+	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	jsonpb "google.golang.org/protobuf/encoding/protojson"
 )
@ -79,6 +80,7 @@ func TestCanDo(t *testing.T) {
 	}
 	// object specific
 	assert.Equal(t, true, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt"))
+	assert.Equal(t, false, ident1.canDo(ACTION_DELETE_BUCKET, "bucket1", ""))
 	assert.Equal(t, false, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/other/some"), "action without *")

 	// bucket specific
@ -141,6 +143,15 @@ func TestCanDo(t *testing.T) {
 		},
 	}
 	assert.Equal(t, true, ident6.canDo(ACTION_READ, "anything_bucket", "/a/b/c/d.txt"))
+
+	//test deleteBucket operation
+	ident7 := &Identity{
+		Name: "anything",
+		Actions: []Action{
+			"DeleteBucket:bucket1",
+		},
+	}
+	assert.Equal(t, true, ident7.canDo(ACTION_DELETE_BUCKET, "bucket1", ""))
 }

 type LoadS3ApiConfigurationTestCase struct {
--- a/weed/s3api/s3_constants/s3_actions.go
+++ b/weed/s3api/s3_constants/s3_actions.go
@ -1,13 +1,14 @@
 package s3_constants

 const (
-	ACTION_READ      = "Read"
-	ACTION_READ_ACP  = "ReadAcp"
-	ACTION_WRITE     = "Write"
-	ACTION_WRITE_ACP = "WriteAcp"
-	ACTION_ADMIN     = "Admin"
-	ACTION_TAGGING   = "Tagging"
-	ACTION_LIST      = "List"
+	ACTION_READ          = "Read"
+	ACTION_READ_ACP      = "ReadAcp"
+	ACTION_WRITE         = "Write"
+	ACTION_WRITE_ACP     = "WriteAcp"
+	ACTION_ADMIN         = "Admin"
+	ACTION_TAGGING       = "Tagging"
+	ACTION_LIST          = "List"
+	ACTION_DELETE_BUCKET = "DeleteBucket"

 	SeaweedStorageDestinationHeader = "x-seaweedfs-destination"
 	MultipartUploadsFolder          = ".uploads"
--- a/weed/s3api/s3_constants/s3_config.go
+++ b/weed/s3api/s3_constants/s3_config.go
@ -7,7 +7,7 @@ import (
 var (
 	CircuitBreakerConfigDir  = "/etc/s3"
 	CircuitBreakerConfigFile = "circuit_breaker.json"
-	AllowedActions           = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN}
+	AllowedActions           = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN, ACTION_DELETE_BUCKET}
 	LimitTypeCount           = "Count"
 	LimitTypeBytes           = "MB"
 	Separator                = ":"
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@ -6,14 +6,15 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
-	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"net/http"
 	"strings"
 	"time"

+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@ -218,6 +219,10 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC
 		return s3err.ErrNoSuchBucket
 	}

+	//if iam is enabled, the access was already checked before
+	if s3a.iam.isEnabled() {
+		return s3err.ErrNone
+	}
 	if !s3a.hasAccess(r, entry) {
 		return s3err.ErrAccessDenied
 	}
@ -236,6 +241,7 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
 	identityId := r.Header.Get(s3_constants.AmzIdentityId)
 	if id, ok := entry.Extended[s3_constants.AmzIdentityId]; ok {
 		if identityId != string(id) {
+			glog.V(3).Infof("hasAccess: %s != %s (entry.Extended = %v)", identityId, id, entry.Extended)
 			return false
 		}
 	}
--- a/weed/s3api/s3api_server.go
+++ b/weed/s3api/s3api_server.go
@ -280,7 +280,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT"))

 		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_ADMIN)), "DELETE"))
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))

 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
--- a/weed/storage/needle_map_memory.go
+++ b/weed/storage/needle_map_memory.go
@ -36,8 +36,8 @@ func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
 func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
 	e := idx.WalkIndexFile(file, 0, func(key NeedleId, offset Offset, size Size) error {
 		nm.MaybeSetMaxFileKey(key)
-		nm.FileCounter++
 		if !offset.IsZero() && size.IsValid() {
+			nm.FileCounter++
 			nm.FileByteCounter = nm.FileByteCounter + uint64(size)
 			oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
 			if !oldOffset.IsZero() && oldSize.IsValid() {
@ -51,7 +51,7 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
 		}
 		return nil
 	})
-	glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name())
+	glog.V(1).Infof("max file key: %v count: %d deleted: %d for file: %s", nm.MaxFileKey(), nm.FileCount(), nm.DeletedCount(), file.Name())
 	return nm, e
 }

--- a/weed/storage/volume_checking.go
+++ b/weed/storage/volume_checking.go
@ -109,9 +109,6 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version,
 			return 0, fmt.Errorf("verifyNeedleIntegrity check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err)
 		}
 		n.AppendAtNs = util.BytesToUint64(bytes)
-		if n.HasTtl() {
-			return n.AppendAtNs, nil
-		}
 		fileTailOffset := offset + needle.GetActualSize(size, v)
 		fileSize, _, err := datFile.GetStat()
 		if err != nil {
@ -130,7 +127,7 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version,
 		return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err)
 	}
 	if n.Id != key {
-		return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
+		return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id)
 	}
 	return n.AppendAtNs, err
 }
@ -147,7 +144,7 @@ func verifyDeletedNeedleIntegrity(datFile backend.BackendStorageFile, v needle.V
 		return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", fileSize-size, size, err)
 	}
 	if n.Id != key {
-		return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
+		return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id)
 	}
 	return n.AppendAtNs, err
 }
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@ -487,19 +487,21 @@ func (v *Volume) copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, da
 	if err != nil {
 		return err
 	}
-	dstDatSize, _, err := dstDatBackend.GetStat()
-	if err != nil {
-		return err
-	}
-	if v.nm.ContentSize() > v.nm.DeletedSize() {
-		expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize()
-		if expectedContentSize > uint64(dstDatSize) {
-			return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d",
-				v.Id.String(), dstDatSize, expectedContentSize)
+	if v.Ttl.String() == "" {
+		dstDatSize, _, err := dstDatBackend.GetStat()
+		if err != nil {
+			return err
+		}
+		if v.nm.ContentSize() > v.nm.DeletedSize() {
+			expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize()
+			if expectedContentSize > uint64(dstDatSize) {
+				return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d",
+					v.Id.String(), dstDatSize, expectedContentSize)
+			}
+		} else {
+			glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d",
+				v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
 		}
-	} else {
-		glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d",
-			v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
 	}
 	err = newNm.SaveToIdx(datIdxName)
 	if err != nil {