Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Object locking need to persist the tags and set the headers (#6994) 

* fix object locking read and write

No logic to include object lock metadata in HEAD/GET response headers
No logic to extract object lock metadata from PUT request headers

* add tests for object locking

* Update weed/s3api/s3api_object_handlers_put.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor

* add unit tests

* sync versions

* Update s3_worm_integration_test.go

* fix legal hold values

* lint

* fix tests

* racing condition when enable versioning

* fix tests

* validate put object lock header

* allow check lock permissions for PUT

* default to OFF legal hold

* only set object lock headers for objects that are actually from object lock-enabled buckets

fix     --- FAIL: TestAddObjectLockHeadersToResponse/Handle_entry_with_no_object_lock_metadata (0.00s)

* address comments

* fix tests

* purge

* fix

* refactoring

* address comment

* address comment

* Update weed/s3api/s3api_object_handlers_put.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers_put.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_handlers.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* avoid nil

* ensure locked objects cannot be overwritten

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
pull/6422/merge
Chris Lu 3 months ago
committed by GitHub
parent
89706d36dc
commit
a524b4f485
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
19 changed files with 1397 additions and 255 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 36
      test/s3/cors/go.mod
  2. 63
      test/s3/cors/go.sum
  3. 14
      test/s3/cors/s3_cors_http_test.go
  4. 28
      test/s3/cors/s3_cors_test.go
  5. 31
      test/s3/retention/go.mod
  6. 62
      test/s3/retention/go.sum
  7. 2
      test/s3/retention/object_lock_reproduce_test.go
  8. 2
      test/s3/retention/object_lock_validation_test.go
  9. 4
      test/s3/retention/s3_bucket_object_lock_test.go
  10. 307
      test/s3/retention/s3_object_lock_headers_test.go
  11. 18
      test/s3/retention/s3_retention_test.go
  12. 14
      test/s3/retention/s3_worm_integration_test.go
  13. 160
      test/s3/versioning/s3_versioning_object_lock_test.go
  14. 16
      weed/notification/webhook/webhook_queue.go
  15. 3
      weed/s3api/s3_constants/header.go
  16. 48
      weed/s3api/s3api_object_handlers.go
  17. 160
      weed/s3api/s3api_object_handlers_put.go
  18. 662
      weed/s3api/s3api_object_lock_headers_test.go
  19. 16
      weed/s3api/s3api_object_retention.go

36
test/s3/cors/go.mod
View File

@ -1,36 +0,0 @@
module github.com/seaweedfs/seaweedfs/test/s3/cors
go 1.19
require (
github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/config v1.18.42
github.com/aws/aws-sdk-go-v2/credentials v1.13.40
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0
github.com/k0kubun/pp v3.0.1+incompatible
github.com/stretchr/testify v1.8.4
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

63
test/s3/cors/go.sum
View File

@ -1,63 +0,0 @@
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
github.com/aws/aws-sdk-go-v2/config v1.18.42 h1:28jHROB27xZwU0CB88giDSjz7M1Sba3olb5JBGwina8=
github.com/aws/aws-sdk-go-v2/config v1.18.42/go.mod h1:4AZM3nMMxwlG+eZlxvBKqwVbkDLlnN2a4UGTL6HjaZI=
github.com/aws/aws-sdk-go-v2/credentials v1.13.40 h1:s8yOkDh+5b1jUDhMBtngF6zKWLDs84chUk2Vk0c38Og=
github.com/aws/aws-sdk-go-v2/credentials v1.13.40/go.mod h1:VtEHVAAqDWASwdOqj/1huyT6uHbs5s8FUHfDQdky/Rs=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 h1:g+qlObJH4Kn4n21g69DjspU0hKTjWtq7naZ9OLCv0ew=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 h1:wl5dxN1NONhTDQD9uaEvNsDRX29cBmGED/nl0jkWlt4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
github.com/aws/aws-sdk-go-v2/service/sso v1.14.1 h1:YkNzx1RLS0F5qdf9v1Q8Cuv9NXCL2TkosOxhzlUPV64=
github.com/aws/aws-sdk-go-v2/service/sso v1.14.1/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1 h1:8lKOidPkmSmfUtiTgtdXWgaKItCZ/g75/jEk6Ql6GsA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4=
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk=
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

14
test/s3/cors/s3_cors_http_test.go
View File

@ -29,7 +29,7 @@ func TestCORSPreflightRequest(t *testing.T) {
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag", "Content-Length"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -81,7 +81,7 @@ func TestCORSActualRequest(t *testing.T) {
AllowedMethods: []string{"GET", "PUT"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag", "Content-Length"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -175,7 +175,7 @@ func TestCORSOriginMatching(t *testing.T) {
AllowedMethods: []string{"GET"},
AllowedOrigins: tc.allowedOrigins,
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -268,7 +268,7 @@ func TestCORSHeaderMatching(t *testing.T) {
AllowedMethods: []string{"GET", "POST"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -349,7 +349,7 @@ func TestCORSMethodMatching(t *testing.T) {
AllowedMethods: []string{"GET", "POST"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -413,14 +413,14 @@ func TestCORSMultipleRulesMatching(t *testing.T) {
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
{
AllowedHeaders: []string{"Authorization"},
AllowedMethods: []string{"POST", "PUT"},
AllowedOrigins: []string{"https://api.example.com"},
ExposeHeaders: []string{"Content-Length"},
MaxAgeSeconds: 7200,
MaxAgeSeconds: aws.Int32(7200),
},
},
}

28
test/s3/cors/s3_cors_test.go
View File

@ -128,7 +128,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
AllowedMethods: []string{"GET", "POST", "PUT"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -152,7 +152,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
assert.Equal(t, []string{"GET", "POST", "PUT"}, rule.AllowedMethods, "Allowed methods should match")
assert.Equal(t, []string{"https://example.com"}, rule.AllowedOrigins, "Allowed origins should match")
assert.Equal(t, []string{"ETag"}, rule.ExposeHeaders, "Expose headers should match")
assert.Equal(t, int32(3600), rule.MaxAgeSeconds, "Max age should match")
assert.Equal(t, aws.Int32(3600), rule.MaxAgeSeconds, "Max age should match")
// Test 4: Update CORS configuration
updatedCorsConfig := &types.CORSConfiguration{
@ -162,7 +162,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
AllowedMethods: []string{"GET", "POST"},
AllowedOrigins: []string{"https://example.com", "https://another.com"},
ExposeHeaders: []string{"ETag", "Content-Length"},
MaxAgeSeconds: 7200,
MaxAgeSeconds: aws.Int32(7200),
},
},
}
@ -209,21 +209,21 @@ func TestCORSMultipleRules(t *testing.T) {
AllowedMethods: []string{"GET", "HEAD"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
{
AllowedHeaders: []string{"Content-Type", "Authorization"},
AllowedMethods: []string{"POST", "PUT", "DELETE"},
AllowedOrigins: []string{"https://app.example.com"},
ExposeHeaders: []string{"ETag", "Content-Length"},
MaxAgeSeconds: 7200,
MaxAgeSeconds: aws.Int32(7200),
},
{
AllowedHeaders: []string{"*"},
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{"*"},
ExposeHeaders: []string{"ETag"},
MaxAgeSeconds: 1800,
MaxAgeSeconds: aws.Int32(1800),
},
},
}
@ -307,7 +307,7 @@ func TestCORSValidation(t *testing.T) {
AllowedHeaders: []string{"*"},
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{"https://example.com"},
MaxAgeSeconds: -1,
MaxAgeSeconds: aws.Int32(-1),
},
},
}
@ -333,7 +333,7 @@ func TestCORSWithWildcards(t *testing.T) {
AllowedMethods: []string{"GET", "POST"},
AllowedOrigins: []string{"https://*.example.com"},
ExposeHeaders: []string{"*"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -370,7 +370,7 @@ func TestCORSRuleLimit(t *testing.T) {
AllowedHeaders: []string{"*"},
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{fmt.Sprintf("https://example%d.com", i)},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
}
}
@ -389,7 +389,7 @@ func TestCORSRuleLimit(t *testing.T) {
AllowedHeaders: []string{"*"},
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{"https://example101.com"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
})
corsConfig.CORSRules = rules
@ -450,7 +450,7 @@ func TestCORSObjectOperations(t *testing.T) {
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"},
AllowedOrigins: []string{"https://example.com"},
ExposeHeaders: []string{"ETag", "Content-Length"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -501,7 +501,7 @@ func TestCORSCaching(t *testing.T) {
AllowedHeaders: []string{"*"},
AllowedMethods: []string{"GET"},
AllowedOrigins: []string{"https://example.com"},
MaxAgeSeconds: 3600,
MaxAgeSeconds: aws.Int32(3600),
},
},
}
@ -526,7 +526,7 @@ func TestCORSCaching(t *testing.T) {
AllowedHeaders: []string{"Content-Type"},
AllowedMethods: []string{"GET", "POST"},
AllowedOrigins: []string{"https://example.com", "https://another.com"},
MaxAgeSeconds: 7200,
MaxAgeSeconds: aws.Int32(7200),
},
},
}
@ -548,7 +548,7 @@ func TestCORSCaching(t *testing.T) {
assert.Equal(t, []string{"Content-Type"}, rule.AllowedHeaders, "Should have updated headers")
assert.Equal(t, []string{"GET", "POST"}, rule.AllowedMethods, "Should have updated methods")
assert.Equal(t, []string{"https://example.com", "https://another.com"}, rule.AllowedOrigins, "Should have updated origins")
assert.Equal(t, int32(7200), rule.MaxAgeSeconds, "Should have updated max age")
assert.Equal(t, aws.Int32(7200), rule.MaxAgeSeconds, "Should have updated max age")
}
// TestCORSErrorHandling tests various error conditions

31
test/s3/retention/go.mod
View File

@ -1,31 +0,0 @@
module github.com/seaweedfs/seaweedfs/test/s3/retention
go 1.21
require (
github.com/aws/aws-sdk-go-v2 v1.21.2
github.com/aws/aws-sdk-go-v2/config v1.18.45
github.com/aws/aws-sdk-go-v2/credentials v1.13.43
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0
github.com/stretchr/testify v1.8.4
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect
github.com/aws/smithy-go v1.15.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

62
test/s3/retention/go.sum
View File

@ -1,62 +0,0 @@
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes=
github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE=
github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8=
github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 h1:wl5dxN1NONhTDQD9uaEvNsDRX29cBmGED/nl0jkWlt4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k=
github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg=
github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU=
github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ=
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

2
test/s3/retention/object_lock_reproduce_test.go
View File

@ -25,7 +25,7 @@ func TestReproduceObjectLockIssue(t *testing.T) {
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This sets the x-amz-bucket-object-lock-enabled header
ObjectLockEnabledForBucket: aws.Bool(true), // This sets the x-amz-bucket-object-lock-enabled header
})
if err != nil {

2
test/s3/retention/object_lock_validation_test.go
View File

@ -26,7 +26,7 @@ func TestObjectLockValidation(t *testing.T) {
t.Log("\n1. Creating bucket with x-amz-bucket-object-lock-enabled: true")
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This sends x-amz-bucket-object-lock-enabled: true
ObjectLockEnabledForBucket: aws.Bool(true), // This sends x-amz-bucket-object-lock-enabled: true
})
require.NoError(t, err, "Bucket creation should succeed")
defer client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})

4
test/s3/retention/s3_bucket_object_lock_test.go
View File

@ -32,7 +32,7 @@ func TestBucketCreationWithObjectLockEnabled(t *testing.T) {
// This simulates what S3 clients do when testing Object Lock support
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true, // This should set x-amz-bucket-object-lock-enabled header
ObjectLockEnabledForBucket: aws.Bool(true), // This should set x-amz-bucket-object-lock-enabled header
})
require.NoError(t, err)
require.NotNil(t, createResp)
@ -122,7 +122,7 @@ func TestS3ObjectLockWorkflow(t *testing.T) {
t.Run("ClientCreatesBucket", func(t *testing.T) {
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: true,
ObjectLockEnabledForBucket: aws.Bool(true),
})
require.NoError(t, err)
})

307
test/s3/retention/s3_object_lock_headers_test.go
View File

@ -0,0 +1,307 @@
package retention
import (
"context"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestPutObjectWithLockHeaders tests that object lock headers in PUT requests
// are properly stored and returned in HEAD responses
func TestPutObjectWithLockHeaders(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create bucket with object lock enabled and versioning
createBucketWithObjectLock(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "test-object-lock-headers"
content := "test content with object lock headers"
retainUntilDate := time.Now().Add(24 * time.Hour)
// Test 1: PUT with COMPLIANCE mode and retention date
t.Run("PUT with COMPLIANCE mode", func(t *testing.T) {
testKey := key + "-compliance"
// PUT object with lock headers
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
"COMPLIANCE", retainUntilDate, "")
require.NotNil(t, putResp.VersionId)
// HEAD object and verify lock headers are returned
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(testKey),
})
require.NoError(t, err)
// Verify object lock metadata is present in response
assert.Equal(t, types.ObjectLockModeCompliance, headResp.ObjectLockMode)
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
})
// Test 2: PUT with GOVERNANCE mode and retention date
t.Run("PUT with GOVERNANCE mode", func(t *testing.T) {
testKey := key + "-governance"
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
"GOVERNANCE", retainUntilDate, "")
require.NotNil(t, putResp.VersionId)
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(testKey),
})
require.NoError(t, err)
assert.Equal(t, types.ObjectLockModeGovernance, headResp.ObjectLockMode)
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
})
// Test 3: PUT with legal hold
t.Run("PUT with legal hold", func(t *testing.T) {
testKey := key + "-legal-hold"
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
"", time.Time{}, "ON")
require.NotNil(t, putResp.VersionId)
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(testKey),
})
require.NoError(t, err)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
})
// Test 4: PUT with both retention and legal hold
t.Run("PUT with both retention and legal hold", func(t *testing.T) {
testKey := key + "-both"
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
"GOVERNANCE", retainUntilDate, "ON")
require.NotNil(t, putResp.VersionId)
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(testKey),
})
require.NoError(t, err)
assert.Equal(t, types.ObjectLockModeGovernance, headResp.ObjectLockMode)
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
})
}
// TestGetObjectWithLockHeaders verifies that GET requests also return object lock metadata
func TestGetObjectWithLockHeaders(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
createBucketWithObjectLock(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "test-get-object-lock"
content := "test content for GET with lock headers"
retainUntilDate := time.Now().Add(24 * time.Hour)
// PUT object with lock headers
putResp := putObjectWithLockHeaders(t, client, bucketName, key, content,
"COMPLIANCE", retainUntilDate, "ON")
require.NotNil(t, putResp.VersionId)
// GET object and verify lock headers are returned
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err)
defer getResp.Body.Close()
// Verify object lock metadata is present in GET response
assert.Equal(t, types.ObjectLockModeCompliance, getResp.ObjectLockMode)
assert.NotNil(t, getResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate, *getResp.ObjectLockRetainUntilDate, 5*time.Second)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, getResp.ObjectLockLegalHoldStatus)
}
// TestVersionedObjectLockHeaders tests object lock headers work with versioned objects
func TestVersionedObjectLockHeaders(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
createBucketWithObjectLock(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "test-versioned-lock"
content1 := "version 1 content"
content2 := "version 2 content"
retainUntilDate1 := time.Now().Add(12 * time.Hour)
retainUntilDate2 := time.Now().Add(24 * time.Hour)
// PUT first version with GOVERNANCE mode
putResp1 := putObjectWithLockHeaders(t, client, bucketName, key, content1,
"GOVERNANCE", retainUntilDate1, "")
require.NotNil(t, putResp1.VersionId)
// PUT second version with COMPLIANCE mode
putResp2 := putObjectWithLockHeaders(t, client, bucketName, key, content2,
"COMPLIANCE", retainUntilDate2, "ON")
require.NotNil(t, putResp2.VersionId)
require.NotEqual(t, *putResp1.VersionId, *putResp2.VersionId)
// HEAD latest version (version 2)
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err)
assert.Equal(t, types.ObjectLockModeCompliance, headResp.ObjectLockMode)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
// HEAD specific version 1
headResp1, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
VersionId: putResp1.VersionId,
})
require.NoError(t, err)
assert.Equal(t, types.ObjectLockModeGovernance, headResp1.ObjectLockMode)
assert.NotEqual(t, types.ObjectLockLegalHoldStatusOn, headResp1.ObjectLockLegalHoldStatus)
}
// TestObjectLockHeadersErrorCases tests various error scenarios
func TestObjectLockHeadersErrorCases(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
createBucketWithObjectLock(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "test-error-cases"
content := "test content for error cases"
// Test 1: Invalid retention mode should be rejected
t.Run("Invalid retention mode", func(t *testing.T) {
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key + "-invalid-mode"),
Body: strings.NewReader(content),
ObjectLockMode: "INVALID_MODE", // Invalid mode
ObjectLockRetainUntilDate: aws.Time(time.Now().Add(24 * time.Hour)),
})
require.Error(t, err)
})
// Test 2: Retention date in the past should be rejected
t.Run("Past retention date", func(t *testing.T) {
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key + "-past-date"),
Body: strings.NewReader(content),
ObjectLockMode: "GOVERNANCE",
ObjectLockRetainUntilDate: aws.Time(time.Now().Add(-24 * time.Hour)), // Past date
})
require.Error(t, err)
})
// Test 3: Mode without date should be rejected
t.Run("Mode without retention date", func(t *testing.T) {
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key + "-no-date"),
Body: strings.NewReader(content),
ObjectLockMode: "GOVERNANCE",
// Missing ObjectLockRetainUntilDate
})
require.Error(t, err)
})
}
// TestObjectLockHeadersNonVersionedBucket tests that object lock fails on non-versioned buckets
func TestObjectLockHeadersNonVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create regular bucket without object lock/versioning
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "test-non-versioned"
content := "test content"
retainUntilDate := time.Now().Add(24 * time.Hour)
// Attempting to PUT with object lock headers should fail
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content),
ObjectLockMode: "GOVERNANCE",
ObjectLockRetainUntilDate: aws.Time(retainUntilDate),
})
require.Error(t, err)
}
// Helper Functions
// putObjectWithLockHeaders puts an object with object lock headers
func putObjectWithLockHeaders(t *testing.T, client *s3.Client, bucketName, key, content string,
mode string, retainUntilDate time.Time, legalHold string) *s3.PutObjectOutput {
input := &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content),
}
// Add retention mode and date if specified
if mode != "" {
switch mode {
case "COMPLIANCE":
input.ObjectLockMode = types.ObjectLockModeCompliance
case "GOVERNANCE":
input.ObjectLockMode = types.ObjectLockModeGovernance
}
if !retainUntilDate.IsZero() {
input.ObjectLockRetainUntilDate = aws.Time(retainUntilDate)
}
}
// Add legal hold if specified
if legalHold != "" {
switch legalHold {
case "ON":
input.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
case "OFF":
input.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
}
}
resp, err := client.PutObject(context.TODO(), input)
require.NoError(t, err)
return resp
}
// createBucketWithObjectLock creates a bucket with object lock enabled
func createBucketWithObjectLock(t *testing.T, client *s3.Client, bucketName string) {
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: aws.Bool(true),
})
require.NoError(t, err)
// Enable versioning (required for object lock)
enableVersioning(t, client, bucketName)
}

18
test/s3/retention/s3_retention_test.go
View File

@ -160,10 +160,10 @@ func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string)
if len(objectsToDelete) > 0 {
_, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucketName),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
Delete: &types.Delete{
Objects: objectsToDelete,
Quiet: true,
Quiet: aws.Bool(true),
},
})
if err != nil {
@ -174,7 +174,7 @@ func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string)
Bucket: aws.String(bucketName),
Key: obj.Key,
VersionId: obj.VersionId,
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
if delErr != nil {
t.Logf("Warning: failed to delete object %s@%s: %v", *obj.Key, *obj.VersionId, delErr)
@ -277,7 +277,7 @@ func TestBasicRetentionWorkflow(t *testing.T) {
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.NoError(t, err)
}
@ -322,7 +322,7 @@ func TestRetentionModeCompliance(t *testing.T) {
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.Error(t, err)
@ -420,7 +420,7 @@ func TestObjectLockConfiguration(t *testing.T) {
Rule: &types.ObjectLockRule{
DefaultRetention: &types.DefaultRetention{
Mode: types.ObjectLockRetentionModeGovernance,
Days: 30,
Days: aws.Int32(30),
},
},
},
@ -513,7 +513,7 @@ func TestRetentionWithVersions(t *testing.T) {
Bucket: aws.String(bucketName),
Key: aws.String(key),
VersionId: putResp1.VersionId,
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.NoError(t, err)
}
@ -562,7 +562,7 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.Error(t, err)
@ -580,7 +580,7 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.NoError(t, err)
}

14
test/s3/retention/s3_worm_integration_test.go
View File

@ -53,7 +53,7 @@ func TestWORMRetentionIntegration(t *testing.T) {
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
})
require.NoError(t, err)
}
@ -190,7 +190,7 @@ func TestRetentionBulkOperations(t *testing.T) {
Bucket: aws.String(bucketName),
Delete: &types.Delete{
Objects: objectsToDelete,
Quiet: false,
Quiet: aws.Bool(false),
},
})
@ -209,10 +209,10 @@ func TestRetentionBulkOperations(t *testing.T) {
// Try bulk delete with bypass - should succeed
_, err = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucketName),
BypassGovernanceRetention: true,
BypassGovernanceRetention: aws.Bool(true),
Delete: &types.Delete{
Objects: objectsToDelete,
Quiet: false,
Quiet: aws.Bool(false),
},
})
if err != nil {
@ -246,7 +246,7 @@ func TestRetentionWithMultipartUpload(t *testing.T) {
uploadResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
PartNumber: 1,
PartNumber: aws.Int32(1),
UploadId: uploadId,
Body: strings.NewReader(partContent),
})
@ -261,7 +261,7 @@ func TestRetentionWithMultipartUpload(t *testing.T) {
Parts: []types.CompletedPart{
{
ETag: uploadResp.ETag,
PartNumber: 1,
PartNumber: aws.Int32(1),
},
},
},
@ -415,7 +415,7 @@ func TestRetentionBucketDefaults(t *testing.T) {
Rule: &types.ObjectLockRule{
DefaultRetention: &types.DefaultRetention{
Mode: types.ObjectLockRetentionModeGovernance,
Days: 1, // 1 day default
Days: aws.Int32(1), // 1 day default
},
},
},

160
test/s3/versioning/s3_versioning_object_lock_test.go
View File

@ -0,0 +1,160 @@
package s3api
import (
"context"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestVersioningWithObjectLockHeaders ensures that versioned objects properly
// handle object lock headers in PUT requests and return them in HEAD/GET responses.
// This test would have caught the bug where object lock metadata was not returned
// in HEAD/GET responses.
func TestVersioningWithObjectLockHeaders(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create bucket with object lock and versioning enabled
createBucketWithObjectLock(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
key := "versioned-object-with-lock"
content1 := "version 1 content"
content2 := "version 2 content"
// PUT first version with object lock headers
retainUntilDate1 := time.Now().Add(12 * time.Hour)
putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content1),
ObjectLockMode: types.ObjectLockModeGovernance,
ObjectLockRetainUntilDate: aws.Time(retainUntilDate1),
})
require.NoError(t, err)
require.NotNil(t, putResp1.VersionId)
// PUT second version with different object lock settings
retainUntilDate2 := time.Now().Add(24 * time.Hour)
putResp2, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: strings.NewReader(content2),
ObjectLockMode: types.ObjectLockModeCompliance,
ObjectLockRetainUntilDate: aws.Time(retainUntilDate2),
ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn,
})
require.NoError(t, err)
require.NotNil(t, putResp2.VersionId)
require.NotEqual(t, *putResp1.VersionId, *putResp2.VersionId)
// Test HEAD latest version returns correct object lock metadata
t.Run("HEAD latest version", func(t *testing.T) {
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err)
// Should return metadata for version 2 (latest)
assert.Equal(t, types.ObjectLockModeCompliance, headResp.ObjectLockMode)
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate2, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
})
// Test HEAD specific version returns correct object lock metadata
t.Run("HEAD specific version", func(t *testing.T) {
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
VersionId: putResp1.VersionId,
})
require.NoError(t, err)
// Should return metadata for version 1
assert.Equal(t, types.ObjectLockModeGovernance, headResp.ObjectLockMode)
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate1, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
// Version 1 was created without legal hold, so AWS S3 defaults it to "OFF"
assert.Equal(t, types.ObjectLockLegalHoldStatusOff, headResp.ObjectLockLegalHoldStatus)
})
// Test GET latest version returns correct object lock metadata
t.Run("GET latest version", func(t *testing.T) {
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err)
defer getResp.Body.Close()
// Should return metadata for version 2 (latest)
assert.Equal(t, types.ObjectLockModeCompliance, getResp.ObjectLockMode)
assert.NotNil(t, getResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate2, *getResp.ObjectLockRetainUntilDate, 5*time.Second)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, getResp.ObjectLockLegalHoldStatus)
})
// Test GET specific version returns correct object lock metadata
t.Run("GET specific version", func(t *testing.T) {
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
VersionId: putResp1.VersionId,
})
require.NoError(t, err)
defer getResp.Body.Close()
// Should return metadata for version 1
assert.Equal(t, types.ObjectLockModeGovernance, getResp.ObjectLockMode)
assert.NotNil(t, getResp.ObjectLockRetainUntilDate)
assert.WithinDuration(t, retainUntilDate1, *getResp.ObjectLockRetainUntilDate, 5*time.Second)
// Version 1 was created without legal hold, so AWS S3 defaults it to "OFF"
assert.Equal(t, types.ObjectLockLegalHoldStatusOff, getResp.ObjectLockLegalHoldStatus)
})
}
// waitForVersioningToBeEnabled polls the bucket versioning status until it's enabled
// This helps avoid race conditions where object lock is configured but versioning
// isn't immediately available
func waitForVersioningToBeEnabled(t *testing.T, client *s3.Client, bucketName string) {
timeout := time.Now().Add(10 * time.Second)
for time.Now().Before(timeout) {
resp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
if err == nil && resp.Status == types.BucketVersioningStatusEnabled {
return // Versioning is enabled
}
time.Sleep(100 * time.Millisecond)
}
t.Fatalf("Timeout waiting for versioning to be enabled on bucket %s", bucketName)
}
// Helper function for creating buckets with object lock enabled
func createBucketWithObjectLock(t *testing.T, client *s3.Client, bucketName string) {
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
ObjectLockEnabledForBucket: aws.Bool(true),
})
require.NoError(t, err)
// Wait for versioning to be automatically enabled by object lock
waitForVersioningToBeEnabled(t, client, bucketName)
// Verify that object lock was actually enabled
t.Logf("Verifying object lock configuration for bucket %s", bucketName)
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "Object lock should be configured for bucket %s", bucketName)
}

16
weed/notification/webhook/webhook_queue.go
View File

@ -196,7 +196,21 @@ func (w *Queue) logDeadLetterMessages() error {
for {
select {
case msg := <-ch:
glog.Errorf("received dead letter message: %s, key: %s", string(msg.Payload), msg.Metadata["key"])
if msg == nil {
glog.Errorf("received nil message from dead letter channel")
continue
}
key := "unknown"
if msg.Metadata != nil {
if keyValue, exists := msg.Metadata["key"]; exists {
key = keyValue
}
}
payload := ""
if msg.Payload != nil {
payload = string(msg.Payload)
}
glog.Errorf("received dead letter message: %s, key: %s", payload, key)
case <-w.ctx.Done():
return
}

3
weed/s3api/s3_constants/header.go
View File

@ -53,6 +53,9 @@ const (
// S3 Object Lock headers
AmzBucketObjectLockEnabled = "X-Amz-Bucket-Object-Lock-Enabled"
AmzObjectLockMode = "X-Amz-Object-Lock-Mode"
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
// S3 conditional copy headers
AmzCopySourceIfMatch = "X-Amz-Copy-Source-If-Match"

48
weed/s3api/s3api_object_handlers.go
View File

@ -6,6 +6,7 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@ -195,6 +196,9 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
// Add object lock metadata to response headers if present
s3a.addObjectLockHeadersToResponse(w, entry)
} else {
// Handle regular GET (non-versioned)
destUrl = s3a.toFilerUrl(bucket, object)
@ -271,6 +275,9 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
// Add object lock metadata to response headers if present
s3a.addObjectLockHeadersToResponse(w, entry)
} else {
// Handle regular HEAD (non-versioned)
destUrl = s3a.toFilerUrl(bucket, object)
@ -435,3 +442,44 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (s
}
return statusCode, bytesTransferred
}
// addObjectLockHeadersToResponse extracts object lock metadata from entry Extended attributes
// and adds the appropriate S3 headers to the response
func (s3a *S3ApiServer) addObjectLockHeadersToResponse(w http.ResponseWriter, entry *filer_pb.Entry) {
if entry == nil || entry.Extended == nil {
return
}
// Check if this entry has any object lock metadata (indicating it's from an object lock enabled bucket)
hasObjectLockMode := false
hasRetentionDate := false
// Add object lock mode header if present
if modeBytes, exists := entry.Extended[s3_constants.ExtObjectLockModeKey]; exists && len(modeBytes) > 0 {
w.Header().Set(s3_constants.AmzObjectLockMode, string(modeBytes))
hasObjectLockMode = true
}
// Add retention until date header if present
if dateBytes, exists := entry.Extended[s3_constants.ExtRetentionUntilDateKey]; exists && len(dateBytes) > 0 {
dateStr := string(dateBytes)
// Convert Unix timestamp to ISO8601 format for S3 compatibility
if timestamp, err := strconv.ParseInt(dateStr, 10, 64); err == nil {
retainUntilDate := time.Unix(timestamp, 0).UTC()
w.Header().Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
hasRetentionDate = true
} else {
glog.Errorf("addObjectLockHeadersToResponse: failed to parse retention until date from stored metadata (dateStr: %s): %v", dateStr, err)
}
}
// Add legal hold header - AWS S3 behavior: always include legal hold for object lock enabled buckets
if legalHoldBytes, exists := entry.Extended[s3_constants.ExtLegalHoldKey]; exists && len(legalHoldBytes) > 0 {
// Return stored S3 standard "ON"/"OFF" values directly
w.Header().Set(s3_constants.AmzObjectLockLegalHold, string(legalHoldBytes))
} else if hasObjectLockMode || hasRetentionDate {
// If this entry has object lock metadata (indicating object lock enabled bucket)
// but no legal hold specifically set, default to "OFF" as per AWS S3 behavior
w.Header().Set(s3_constants.AmzObjectLockLegalHold, s3_constants.LegalHoldOff)
}
}

160
weed/s3api/s3api_object_handlers_put.go
View File

@ -3,9 +3,11 @@ package s3api
import (
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
@ -20,6 +22,18 @@ import (
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
)
// Object lock validation errors
var (
ErrObjectLockVersioningRequired = errors.New("object lock headers can only be used on versioned buckets")
ErrInvalidObjectLockMode = errors.New("invalid object lock mode")
ErrInvalidLegalHoldStatus = errors.New("invalid legal hold status")
ErrInvalidRetentionDateFormat = errors.New("invalid retention until date format")
ErrRetentionDateMustBeFuture = errors.New("retention until date must be in the future")
ErrObjectLockModeRequiresDate = errors.New("object lock mode requires retention until date")
ErrRetentionDateRequiresMode = errors.New("retention until date requires object lock mode")
ErrGovernanceBypassVersioningRequired = errors.New("governance bypass header can only be used on versioned buckets")
)
func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
@ -85,12 +99,23 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningEnabled=%v", bucket, object, versioningEnabled)
// Check object lock permissions before PUT operation (only for versioned buckets)
// Validate object lock headers before processing
if err := s3a.validateObjectLockHeaders(r, versioningEnabled); err != nil {
glog.V(2).Infof("PutObjectHandler: object lock header validation failed for bucket %s, object %s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, mapValidationErrorToS3Error(err))
return
}
// For non-versioned buckets, check if existing object has object lock protections
// that would prevent overwrite (PUT operations overwrite existing objects in non-versioned buckets)
if !versioningEnabled {
bypassGovernance := r.Header.Get("x-amz-bypass-governance-retention") == "true"
if err := s3a.checkObjectLockPermissionsForPut(r, bucket, object, bypassGovernance, versioningEnabled); err != nil {
if err := s3a.checkObjectLockPermissions(r, bucket, object, "", bypassGovernance); err != nil {
glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
}
if versioningEnabled {
// Handle versioned PUT
@ -287,6 +312,12 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin
}
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
// Extract and store object lock metadata from request headers
if err := s3a.extractObjectLockMetadataFromRequest(r, versionEntry); err != nil {
glog.Errorf("putVersionedObject: failed to extract object lock metadata: %v", err)
return "", "", s3err.ErrInvalidRequest
}
// Update the version entry with metadata
err = s3a.mkFile(bucketDir, versionObjectPath, versionEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = versionEntry.Extended
@ -341,3 +372,128 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId
return nil
}
// extractObjectLockMetadataFromRequest extracts object lock headers from PUT requests
// and stores them in the entry's Extended attributes
func (s3a *S3ApiServer) extractObjectLockMetadataFromRequest(r *http.Request, entry *filer_pb.Entry) error {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
// Extract object lock mode (GOVERNANCE or COMPLIANCE)
if mode := r.Header.Get(s3_constants.AmzObjectLockMode); mode != "" {
entry.Extended[s3_constants.ExtObjectLockModeKey] = []byte(mode)
glog.V(2).Infof("extractObjectLockMetadataFromRequest: storing object lock mode: %s", mode)
}
// Extract retention until date
if retainUntilDate := r.Header.Get(s3_constants.AmzObjectLockRetainUntilDate); retainUntilDate != "" {
// Parse the ISO8601 date and convert to Unix timestamp for storage
parsedTime, err := time.Parse(time.RFC3339, retainUntilDate)
if err != nil {
glog.Errorf("extractObjectLockMetadataFromRequest: failed to parse retention until date, expected format: %s, error: %v", time.RFC3339, err)
return ErrInvalidRetentionDateFormat
}
entry.Extended[s3_constants.ExtRetentionUntilDateKey] = []byte(strconv.FormatInt(parsedTime.Unix(), 10))
glog.V(2).Infof("extractObjectLockMetadataFromRequest: storing retention until date (timestamp: %d)", parsedTime.Unix())
}
// Extract legal hold status
if legalHold := r.Header.Get(s3_constants.AmzObjectLockLegalHold); legalHold != "" {
// Store S3 standard "ON"/"OFF" values directly
if legalHold == s3_constants.LegalHoldOn || legalHold == s3_constants.LegalHoldOff {
entry.Extended[s3_constants.ExtLegalHoldKey] = []byte(legalHold)
glog.V(2).Infof("extractObjectLockMetadataFromRequest: storing legal hold: %s", legalHold)
} else {
glog.Errorf("extractObjectLockMetadataFromRequest: unexpected legal hold value provided, expected 'ON' or 'OFF'")
return ErrInvalidLegalHoldStatus
}
}
return nil
}
// validateObjectLockHeaders validates object lock headers in PUT requests
func (s3a *S3ApiServer) validateObjectLockHeaders(r *http.Request, versioningEnabled bool) error {
// Extract object lock headers from request
mode := r.Header.Get(s3_constants.AmzObjectLockMode)
retainUntilDateStr := r.Header.Get(s3_constants.AmzObjectLockRetainUntilDate)
legalHold := r.Header.Get(s3_constants.AmzObjectLockLegalHold)
// Check if any object lock headers are present
hasObjectLockHeaders := mode != "" || retainUntilDateStr != "" || legalHold != ""
// Object lock headers can only be used on versioned buckets
if hasObjectLockHeaders && !versioningEnabled {
return ErrObjectLockVersioningRequired
}
// Validate object lock mode if present
if mode != "" {
if mode != s3_constants.RetentionModeGovernance && mode != s3_constants.RetentionModeCompliance {
return ErrInvalidObjectLockMode
}
}
// Validate retention date if present
if retainUntilDateStr != "" {
retainUntilDate, err := time.Parse(time.RFC3339, retainUntilDateStr)
if err != nil {
return ErrInvalidRetentionDateFormat
}
// Retention date must be in the future
if retainUntilDate.Before(time.Now()) {
return ErrRetentionDateMustBeFuture
}
}
// If mode is specified, retention date must also be specified
if mode != "" && retainUntilDateStr == "" {
return ErrObjectLockModeRequiresDate
}
// If retention date is specified, mode must also be specified
if retainUntilDateStr != "" && mode == "" {
return ErrRetentionDateRequiresMode
}
// Validate legal hold if present
if legalHold != "" {
if legalHold != s3_constants.LegalHoldOn && legalHold != s3_constants.LegalHoldOff {
return ErrInvalidLegalHoldStatus
}
}
// Check for governance bypass header - only valid for versioned buckets
bypassGovernance := r.Header.Get("x-amz-bypass-governance-retention") == "true"
// Governance bypass headers are only valid for versioned buckets (like object lock headers)
if bypassGovernance && !versioningEnabled {
return ErrGovernanceBypassVersioningRequired
}
return nil
}
// mapValidationErrorToS3Error maps object lock validation errors to appropriate S3 error codes
func mapValidationErrorToS3Error(err error) s3err.ErrorCode {
switch {
case errors.Is(err, ErrObjectLockVersioningRequired):
return s3err.ErrInvalidRequest
case errors.Is(err, ErrInvalidObjectLockMode):
return s3err.ErrInvalidRequest
case errors.Is(err, ErrInvalidLegalHoldStatus):
return s3err.ErrInvalidRequest
case errors.Is(err, ErrInvalidRetentionDateFormat):
return s3err.ErrMalformedDate
case errors.Is(err, ErrRetentionDateMustBeFuture),
errors.Is(err, ErrObjectLockModeRequiresDate),
errors.Is(err, ErrRetentionDateRequiresMode):
return s3err.ErrInvalidRequest
case errors.Is(err, ErrGovernanceBypassVersioningRequired):
return s3err.ErrInvalidRequest
default:
return s3err.ErrInvalidRequest
}
}

662
weed/s3api/s3api_object_lock_headers_test.go
View File

@ -0,0 +1,662 @@
package s3api
import (
"fmt"
"net/http/httptest"
"strconv"
"testing"
"time"
"errors"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
)
// TestExtractObjectLockMetadataFromRequest tests the function that extracts
// object lock headers from PUT requests and stores them in Extended attributes.
// This test would have caught the bug where object lock headers were ignored.
func TestExtractObjectLockMetadataFromRequest(t *testing.T) {
s3a := &S3ApiServer{}
t.Run("Extract COMPLIANCE mode and retention date", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
// Verify mode was stored
assert.Contains(t, entry.Extended, s3_constants.ExtObjectLockModeKey)
assert.Equal(t, "COMPLIANCE", string(entry.Extended[s3_constants.ExtObjectLockModeKey]))
// Verify retention date was stored
assert.Contains(t, entry.Extended, s3_constants.ExtRetentionUntilDateKey)
storedTimestamp, err := strconv.ParseInt(string(entry.Extended[s3_constants.ExtRetentionUntilDateKey]), 10, 64)
assert.NoError(t, err)
storedTime := time.Unix(storedTimestamp, 0)
assert.WithinDuration(t, retainUntilDate, storedTime, 1*time.Second)
})
t.Run("Extract GOVERNANCE mode and retention date", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(12 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "GOVERNANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
assert.Equal(t, "GOVERNANCE", string(entry.Extended[s3_constants.ExtObjectLockModeKey]))
assert.Contains(t, entry.Extended, s3_constants.ExtRetentionUntilDateKey)
})
t.Run("Extract legal hold ON", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "ON")
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
assert.Contains(t, entry.Extended, s3_constants.ExtLegalHoldKey)
assert.Equal(t, "ON", string(entry.Extended[s3_constants.ExtLegalHoldKey]))
})
t.Run("Extract legal hold OFF", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "OFF")
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
assert.Contains(t, entry.Extended, s3_constants.ExtLegalHoldKey)
assert.Equal(t, "OFF", string(entry.Extended[s3_constants.ExtLegalHoldKey]))
})
t.Run("Handle all object lock headers together", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "ON")
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
// All metadata should be stored
assert.Equal(t, "COMPLIANCE", string(entry.Extended[s3_constants.ExtObjectLockModeKey]))
assert.Contains(t, entry.Extended, s3_constants.ExtRetentionUntilDateKey)
assert.Equal(t, "ON", string(entry.Extended[s3_constants.ExtLegalHoldKey]))
})
t.Run("Handle no object lock headers", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
// No object lock headers set
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
// No object lock metadata should be stored
assert.NotContains(t, entry.Extended, s3_constants.ExtObjectLockModeKey)
assert.NotContains(t, entry.Extended, s3_constants.ExtRetentionUntilDateKey)
assert.NotContains(t, entry.Extended, s3_constants.ExtLegalHoldKey)
})
t.Run("Handle invalid retention date - should return error", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "GOVERNANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, "invalid-date")
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrInvalidRetentionDateFormat))
// Mode should be stored but not invalid date
assert.Equal(t, "GOVERNANCE", string(entry.Extended[s3_constants.ExtObjectLockModeKey]))
assert.NotContains(t, entry.Extended, s3_constants.ExtRetentionUntilDateKey)
})
t.Run("Handle invalid legal hold value - should return error", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "INVALID")
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrInvalidLegalHoldStatus))
// No legal hold metadata should be stored due to error
assert.NotContains(t, entry.Extended, s3_constants.ExtLegalHoldKey)
})
}
// TestAddObjectLockHeadersToResponse tests the function that adds object lock
// metadata from Extended attributes to HTTP response headers.
// This test would have caught the bug where HEAD responses didn't include object lock metadata.
func TestAddObjectLockHeadersToResponse(t *testing.T) {
s3a := &S3ApiServer{}
t.Run("Add COMPLIANCE mode and retention date to response", func(t *testing.T) {
w := httptest.NewRecorder()
retainUntilTime := time.Now().Add(24 * time.Hour)
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtObjectLockModeKey: []byte("COMPLIANCE"),
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(retainUntilTime.Unix(), 10)),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// Verify headers were set
assert.Equal(t, "COMPLIANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.NotEmpty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
// Verify the date format is correct
returnedDate := w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate)
parsedTime, err := time.Parse(time.RFC3339, returnedDate)
assert.NoError(t, err)
assert.WithinDuration(t, retainUntilTime, parsedTime, 1*time.Second)
})
t.Run("Add GOVERNANCE mode to response", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtObjectLockModeKey: []byte("GOVERNANCE"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
assert.Equal(t, "GOVERNANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
})
t.Run("Add legal hold ON to response", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtLegalHoldKey: []byte("ON"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
assert.Equal(t, "ON", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Add legal hold OFF to response", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtLegalHoldKey: []byte("OFF"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
assert.Equal(t, "OFF", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Add all object lock headers to response", func(t *testing.T) {
w := httptest.NewRecorder()
retainUntilTime := time.Now().Add(12 * time.Hour)
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtObjectLockModeKey: []byte("GOVERNANCE"),
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(retainUntilTime.Unix(), 10)),
s3_constants.ExtLegalHoldKey: []byte("ON"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// All headers should be set
assert.Equal(t, "GOVERNANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.NotEmpty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Equal(t, "ON", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle entry with no object lock metadata", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
"other-metadata": []byte("some-value"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// No object lock headers should be set for entries without object lock metadata
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle entry with object lock mode but no legal hold - should default to OFF", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtObjectLockModeKey: []byte("GOVERNANCE"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// Should set mode and default legal hold to OFF
assert.Equal(t, "GOVERNANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Equal(t, "OFF", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle entry with retention date but no legal hold - should default to OFF", func(t *testing.T) {
w := httptest.NewRecorder()
retainUntilTime := time.Now().Add(24 * time.Hour)
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(retainUntilTime.Unix(), 10)),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// Should set retention date and default legal hold to OFF
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockMode))
assert.NotEmpty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Equal(t, "OFF", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle nil entry gracefully", func(t *testing.T) {
w := httptest.NewRecorder()
// Should not panic
s3a.addObjectLockHeadersToResponse(w, nil)
// No headers should be set
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle entry with nil Extended map gracefully", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: nil,
}
// Should not panic
s3a.addObjectLockHeadersToResponse(w, entry)
// No headers should be set
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockLegalHold))
})
t.Run("Handle invalid retention timestamp gracefully", func(t *testing.T) {
w := httptest.NewRecorder()
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtObjectLockModeKey: []byte("COMPLIANCE"),
s3_constants.ExtRetentionUntilDateKey: []byte("invalid-timestamp"),
},
}
s3a.addObjectLockHeadersToResponse(w, entry)
// Mode should be set but not retention date due to invalid timestamp
assert.Equal(t, "COMPLIANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Empty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
})
}
// TestObjectLockHeaderRoundTrip tests the complete round trip:
// extract from request → store in Extended attributes → add to response
func TestObjectLockHeaderRoundTrip(t *testing.T) {
s3a := &S3ApiServer{}
t.Run("Complete round trip for COMPLIANCE mode", func(t *testing.T) {
// 1. Create request with object lock headers
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "ON")
// 2. Extract and store in Extended attributes
entry := &filer_pb.Entry{
Extended: make(map[string][]byte),
}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
// 3. Add to response headers
w := httptest.NewRecorder()
s3a.addObjectLockHeadersToResponse(w, entry)
// 4. Verify round trip preserved all data
assert.Equal(t, "COMPLIANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.Equal(t, "ON", w.Header().Get(s3_constants.AmzObjectLockLegalHold))
returnedDate := w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate)
parsedTime, err := time.Parse(time.RFC3339, returnedDate)
assert.NoError(t, err)
assert.WithinDuration(t, retainUntilDate, parsedTime, 1*time.Second)
})
t.Run("Complete round trip for GOVERNANCE mode", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(12 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "GOVERNANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
entry := &filer_pb.Entry{Extended: make(map[string][]byte)}
err := s3a.extractObjectLockMetadataFromRequest(req, entry)
assert.NoError(t, err)
w := httptest.NewRecorder()
s3a.addObjectLockHeadersToResponse(w, entry)
assert.Equal(t, "GOVERNANCE", w.Header().Get(s3_constants.AmzObjectLockMode))
assert.NotEmpty(t, w.Header().Get(s3_constants.AmzObjectLockRetainUntilDate))
})
}
// TestValidateObjectLockHeaders tests the validateObjectLockHeaders function
// to ensure proper validation of object lock headers in PUT requests
func TestValidateObjectLockHeaders(t *testing.T) {
s3a := &S3ApiServer{}
t.Run("Valid COMPLIANCE mode with retention date on versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("Valid GOVERNANCE mode with retention date on versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(12 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "GOVERNANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("Valid legal hold ON on versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "ON")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("Valid legal hold OFF on versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "OFF")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("Invalid object lock mode", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "INVALID_MODE")
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrInvalidObjectLockMode))
})
t.Run("Invalid legal hold status", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "INVALID_STATUS")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrInvalidLegalHoldStatus))
})
t.Run("Object lock headers on non-versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, false) // non-versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrObjectLockVersioningRequired))
})
t.Run("Invalid retention date format", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, "invalid-date-format")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrInvalidRetentionDateFormat))
})
t.Run("Retention date in the past", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
pastDate := time.Now().Add(-24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, pastDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrRetentionDateMustBeFuture))
})
t.Run("Mode without retention date", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzObjectLockMode, "COMPLIANCE")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrObjectLockModeRequiresDate))
})
t.Run("Retention date without mode", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(24 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrRetentionDateRequiresMode))
})
t.Run("Governance bypass header on non-versioned bucket", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set("x-amz-bypass-governance-retention", "true")
err := s3a.validateObjectLockHeaders(req, false) // non-versioned bucket
assert.Error(t, err)
assert.True(t, errors.Is(err, ErrGovernanceBypassVersioningRequired))
})
t.Run("Governance bypass header on versioned bucket should pass", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
req.Header.Set("x-amz-bypass-governance-retention", "true")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("No object lock headers should pass", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
// No object lock headers set
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
t.Run("Mixed valid headers should pass", func(t *testing.T) {
req := httptest.NewRequest("PUT", "/bucket/object", nil)
retainUntilDate := time.Now().Add(48 * time.Hour)
req.Header.Set(s3_constants.AmzObjectLockMode, "GOVERNANCE")
req.Header.Set(s3_constants.AmzObjectLockRetainUntilDate, retainUntilDate.Format(time.RFC3339))
req.Header.Set(s3_constants.AmzObjectLockLegalHold, "ON")
err := s3a.validateObjectLockHeaders(req, true) // versioned bucket
assert.NoError(t, err)
})
}
// TestMapValidationErrorToS3Error tests the error mapping function
func TestMapValidationErrorToS3Error(t *testing.T) {
tests := []struct {
name string
inputError error
expectedCode s3err.ErrorCode
}{
{
name: "ErrObjectLockVersioningRequired",
inputError: ErrObjectLockVersioningRequired,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrInvalidObjectLockMode",
inputError: ErrInvalidObjectLockMode,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrInvalidLegalHoldStatus",
inputError: ErrInvalidLegalHoldStatus,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrInvalidRetentionDateFormat",
inputError: ErrInvalidRetentionDateFormat,
expectedCode: s3err.ErrMalformedDate,
},
{
name: "ErrRetentionDateMustBeFuture",
inputError: ErrRetentionDateMustBeFuture,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrObjectLockModeRequiresDate",
inputError: ErrObjectLockModeRequiresDate,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrRetentionDateRequiresMode",
inputError: ErrRetentionDateRequiresMode,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "ErrGovernanceBypassVersioningRequired",
inputError: ErrGovernanceBypassVersioningRequired,
expectedCode: s3err.ErrInvalidRequest,
},
{
name: "Unknown error defaults to ErrInvalidRequest",
inputError: fmt.Errorf("unknown error"),
expectedCode: s3err.ErrInvalidRequest,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := mapValidationErrorToS3Error(tt.inputError)
assert.Equal(t, tt.expectedCode, result)
})
}
}
// TestObjectLockPermissionLogic documents the correct behavior for object lock permission checks
// in PUT operations for both versioned and non-versioned buckets
func TestObjectLockPermissionLogic(t *testing.T) {
t.Run("Non-versioned bucket PUT operation logic", func(t *testing.T) {
// In non-versioned buckets, PUT operations overwrite existing objects
// Therefore, we MUST check if the existing object has object lock protections
// that would prevent overwrite before allowing the PUT operation.
//
// This test documents the expected behavior:
// 1. Check object lock headers validity (handled by validateObjectLockHeaders)
// 2. Check if existing object has object lock protections (handled by checkObjectLockPermissions)
// 3. If existing object is under retention/legal hold, deny the PUT unless governance bypass is valid
t.Log("For non-versioned buckets:")
t.Log("- PUT operations overwrite existing objects")
t.Log("- Must check existing object lock protections before allowing overwrite")
t.Log("- Governance bypass headers can be used to override GOVERNANCE mode retention")
t.Log("- COMPLIANCE mode retention and legal holds cannot be bypassed")
})
t.Run("Versioned bucket PUT operation logic", func(t *testing.T) {
// In versioned buckets, PUT operations create new versions without overwriting existing ones
// Therefore, we do NOT need to check existing object permissions since we're not modifying them.
// We only need to validate the object lock headers for the new version being created.
//
// This test documents the expected behavior:
// 1. Check object lock headers validity (handled by validateObjectLockHeaders)
// 2. Skip checking existing object permissions (since we're creating a new version)
// 3. Apply object lock metadata to the new version being created
t.Log("For versioned buckets:")
t.Log("- PUT operations create new versions without overwriting existing objects")
t.Log("- No need to check existing object lock protections")
t.Log("- Only validate object lock headers for the new version being created")
t.Log("- Each version has independent object lock settings")
})
t.Run("Governance bypass header validation", func(t *testing.T) {
// Governance bypass headers should only be used in specific scenarios:
// 1. Only valid on versioned buckets (consistent with object lock headers)
// 2. For non-versioned buckets: Used to override existing object's GOVERNANCE retention
// 3. For versioned buckets: Not typically needed since new versions don't conflict with existing ones
t.Log("Governance bypass behavior:")
t.Log("- Only valid on versioned buckets (header validation)")
t.Log("- For non-versioned buckets: Allows overwriting objects under GOVERNANCE retention")
t.Log("- For versioned buckets: Not typically needed for PUT operations")
t.Log("- Must have s3:BypassGovernanceRetention permission")
})
}

16
weed/s3api/s3api_object_retention.go
View File

@ -611,22 +611,6 @@ func (s3a *S3ApiServer) isObjectLockAvailable(bucket string) error {
return nil
}
// checkObjectLockPermissionsForPut checks object lock permissions for PUT operations
// This is a shared helper to avoid code duplication in PUT handlers
func (s3a *S3ApiServer) checkObjectLockPermissionsForPut(request *http.Request, bucket, object string, bypassGovernance bool, versioningEnabled bool) error {
// Object Lock only applies to versioned buckets (AWS S3 requirement)
if !versioningEnabled {
return nil
}
// For PUT operations, we check permissions on the current object (empty versionId)
if err := s3a.checkObjectLockPermissions(request, bucket, object, "", bypassGovernance); err != nil {
glog.V(2).Infof("checkObjectLockPermissionsForPut: object lock check failed for %s/%s: %v", bucket, object, err)
return err
}
return nil
}
// handleObjectLockAvailabilityCheck is a helper function to check object lock availability
// and write the appropriate error response if not available. This reduces code duplication
// across all retention handlers.

Write Preview
Loading…
Cancel
Save