diff --git a/.gitignore b/.gitignore index 8d3a78ba5..215cb827b 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ test/s3/cors/weed-test.log test/s3/cors/weed-server.pid /test/s3/cors/test-volume-data test/s3/cors/cors.test +/test/s3/retention/filerldb2 +test/s3/retention/weed-test.log +test/s3/retention/weed-server.pid diff --git a/test/s3/retention/object_lock_reproduce_test.go b/test/s3/retention/object_lock_reproduce_test.go new file mode 100644 index 000000000..41115ba47 --- /dev/null +++ b/test/s3/retention/object_lock_reproduce_test.go @@ -0,0 +1,114 @@ +package retention + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" +) + +// TestReproduceObjectLockIssue reproduces the Object Lock header processing issue step by step +func TestReproduceObjectLockIssue(t *testing.T) { + client := getS3Client(t) + bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano()) + + t.Logf("=== Reproducing Object Lock Header Processing Issue ===") + t.Logf("Bucket name: %s", bucketName) + + // Step 1: Create bucket with Object Lock enabled header + t.Logf("\n1. Creating bucket with ObjectLockEnabledForBucket=true") + t.Logf(" This should send x-amz-bucket-object-lock-enabled: true header") + + createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + ObjectLockEnabledForBucket: true, // This sets the x-amz-bucket-object-lock-enabled header + }) + + if err != nil { + t.Fatalf("Bucket creation failed: %v", err) + } + t.Logf("āœ… Bucket created successfully") + t.Logf(" Response: %+v", createResp) + + // Step 2: Check if Object Lock is actually enabled + t.Logf("\n2. Checking Object Lock configuration to verify it was enabled") + + objectLockResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + + if err != nil { + t.Logf("āŒ GetObjectLockConfiguration FAILED: %v", err) + t.Logf(" This demonstrates the issue with header processing!") + t.Logf(" S3 clients expect this call to succeed if Object Lock is supported") + t.Logf(" When this fails, clients conclude that Object Lock is not supported") + + // This failure demonstrates the bug - the bucket was created but Object Lock wasn't enabled + t.Logf("\nšŸ› BUG CONFIRMED:") + t.Logf(" - Bucket creation with ObjectLockEnabledForBucket=true succeeded") + t.Logf(" - But GetObjectLockConfiguration fails") + t.Logf(" - This means the x-amz-bucket-object-lock-enabled header was ignored") + + } else { + t.Logf("āœ… GetObjectLockConfiguration succeeded!") + t.Logf(" Response: %+v", objectLockResp) + t.Logf(" Object Lock is properly enabled - this is the expected behavior") + } + + // Step 3: Check versioning status (required for Object Lock) + t.Logf("\n3. Checking bucket versioning status (required for Object Lock)") + + versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + t.Logf(" Versioning status: %v", versioningResp.Status) + if versioningResp.Status != "Enabled" { + t.Logf(" āš ļø Versioning should be automatically enabled when Object Lock is enabled") + } + + // Cleanup + t.Logf("\n4. Cleaning up test bucket") + _, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + if err != nil { + t.Logf(" Warning: Failed to delete bucket: %v", err) + } + + t.Logf("\n=== Issue Reproduction Complete ===") + t.Logf("Expected behavior after fix:") + t.Logf(" - CreateBucket with ObjectLockEnabledForBucket=true should enable Object Lock") + t.Logf(" - GetObjectLockConfiguration should return enabled configuration") + t.Logf(" - Versioning should be automatically enabled") +} + +// TestNormalBucketCreationStillWorks tests that normal bucket creation still works +func TestNormalBucketCreationStillWorks(t *testing.T) { + client := getS3Client(t) + bucketName := fmt.Sprintf("normal-test-%d", time.Now().UnixNano()) + + t.Logf("=== Testing Normal Bucket Creation ===") + + // Create bucket without Object Lock + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + t.Logf("āœ… Normal bucket creation works") + + // Object Lock should NOT be enabled + _, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + require.Error(t, err, "GetObjectLockConfiguration should fail for bucket without Object Lock") + t.Logf("āœ… GetObjectLockConfiguration correctly fails for normal bucket") + + // Cleanup + client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)}) +} diff --git a/test/s3/retention/object_lock_validation_test.go b/test/s3/retention/object_lock_validation_test.go new file mode 100644 index 000000000..f98aa3b60 --- /dev/null +++ b/test/s3/retention/object_lock_validation_test.go @@ -0,0 +1,105 @@ +package retention + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/require" +) + +// TestObjectLockValidation tests that S3 Object Lock functionality works end-to-end +// This test focuses on the complete Object Lock workflow that S3 clients expect +func TestObjectLockValidation(t *testing.T) { + client := getS3Client(t) + bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano()) + + t.Logf("=== Validating S3 Object Lock Functionality ===") + t.Logf("Bucket: %s", bucketName) + + // Step 1: Create bucket with Object Lock header + t.Log("\n1. Creating bucket with x-amz-bucket-object-lock-enabled: true") + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + ObjectLockEnabledForBucket: true, // This sends x-amz-bucket-object-lock-enabled: true + }) + require.NoError(t, err, "Bucket creation should succeed") + defer client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)}) + t.Log(" āœ… Bucket created successfully") + + // Step 2: Check if Object Lock is supported (standard S3 client behavior) + t.Log("\n2. Testing Object Lock support detection") + _, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err, "GetObjectLockConfiguration should succeed for Object Lock enabled bucket") + t.Log(" āœ… GetObjectLockConfiguration succeeded - Object Lock is properly enabled") + + // Step 3: Verify versioning is enabled (required for Object Lock) + t.Log("\n3. Verifying versioning is automatically enabled") + versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled") + t.Log(" āœ… Versioning automatically enabled") + + // Step 4: Test actual Object Lock functionality + t.Log("\n4. Testing Object Lock retention functionality") + + // Create an object + key := "protected-object.dat" + content := "Important data that needs immutable protection" + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: strings.NewReader(content), + }) + require.NoError(t, err) + require.NotNil(t, putResp.VersionId, "Object should have a version ID") + t.Log(" āœ… Object created with versioning") + + // Apply Object Lock retention + retentionUntil := time.Now().Add(24 * time.Hour) + _, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err, "Setting Object Lock retention should succeed") + t.Log(" āœ… Object Lock retention applied successfully") + + // Verify retention is in effect + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err, "Object should be protected by retention and cannot be deleted") + t.Log(" āœ… Object is properly protected by retention policy") + + // Verify we can read the object (should still work) + getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.NoError(t, err, "Reading protected object should still work") + defer getResp.Body.Close() + t.Log(" āœ… Protected object can still be read") + + t.Log("\nšŸŽ‰ S3 OBJECT LOCK VALIDATION SUCCESSFUL!") + t.Log(" - Bucket creation with Object Lock header works") + t.Log(" - Object Lock support detection works (GetObjectLockConfiguration succeeds)") + t.Log(" - Versioning is automatically enabled") + t.Log(" - Object Lock retention functionality works") + t.Log(" - Objects are properly protected from deletion") + t.Log("") + t.Log("āœ… S3 clients will now recognize SeaweedFS as supporting Object Lock!") +} diff --git a/test/s3/retention/s3_bucket_object_lock_test.go b/test/s3/retention/s3_bucket_object_lock_test.go new file mode 100644 index 000000000..ddbea5eae --- /dev/null +++ b/test/s3/retention/s3_bucket_object_lock_test.go @@ -0,0 +1,185 @@ +package retention + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBucketCreationWithObjectLockEnabled tests creating a bucket with the +// x-amz-bucket-object-lock-enabled header, which is required for S3 Object Lock compatibility +func TestBucketCreationWithObjectLockEnabled(t *testing.T) { + // This test verifies that bucket creation with + // x-amz-bucket-object-lock-enabled header should automatically enable Object Lock + + client := getS3Client(t) + bucketName := getNewBucketName() + defer func() { + // Best effort cleanup + deleteBucket(t, client, bucketName) + }() + + // Test 1: Create bucket with Object Lock enabled header using custom HTTP client + t.Run("CreateBucketWithObjectLockHeader", func(t *testing.T) { + // Create bucket with x-amz-bucket-object-lock-enabled header + // This simulates what S3 clients do when testing Object Lock support + createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + ObjectLockEnabledForBucket: true, // This should set x-amz-bucket-object-lock-enabled header + }) + require.NoError(t, err) + require.NotNil(t, createResp) + + // Verify bucket was created + _, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + }) + + // Test 2: Verify that Object Lock is automatically enabled for the bucket + t.Run("VerifyObjectLockAutoEnabled", func(t *testing.T) { + // Try to get the Object Lock configuration + // If the header was processed correctly, this should return an enabled configuration + configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + + require.NoError(t, err, "GetObjectLockConfiguration should not fail if Object Lock is enabled") + require.NotNil(t, configResp.ObjectLockConfiguration, "ObjectLockConfiguration should not be nil") + assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled, "Object Lock should be enabled") + }) + + // Test 3: Verify versioning is automatically enabled (required for Object Lock) + t.Run("VerifyVersioningAutoEnabled", func(t *testing.T) { + // Object Lock requires versioning to be enabled + // When Object Lock is enabled via header, versioning should also be enabled automatically + versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + // Versioning should be automatically enabled for Object Lock + assert.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled for Object Lock") + }) +} + +// TestBucketCreationWithoutObjectLockHeader tests normal bucket creation +// to ensure we don't break existing functionality +func TestBucketCreationWithoutObjectLockHeader(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + defer deleteBucket(t, client, bucketName) + + // Create bucket without Object Lock header + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + // Verify bucket was created + _, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + // Object Lock should NOT be enabled + _, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + // This should fail since Object Lock is not enabled + require.Error(t, err) + t.Logf("GetObjectLockConfiguration correctly failed for bucket without Object Lock: %v", err) + + // Versioning should not be enabled by default + versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + // Should be either empty/unset or Suspended, but not Enabled + if versioningResp.Status != types.BucketVersioningStatusEnabled { + t.Logf("Versioning correctly not enabled: %v", versioningResp.Status) + } else { + t.Errorf("Versioning should not be enabled for bucket without Object Lock header") + } +} + +// TestS3ObjectLockWorkflow tests the complete Object Lock workflow that S3 clients would use +func TestS3ObjectLockWorkflow(t *testing.T) { + client := getS3Client(t) + bucketName := getNewBucketName() + defer deleteBucket(t, client, bucketName) + + // Step 1: Client creates bucket with Object Lock enabled + t.Run("ClientCreatesBucket", func(t *testing.T) { + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + ObjectLockEnabledForBucket: true, + }) + require.NoError(t, err) + }) + + // Step 2: Client checks if Object Lock is supported by getting the configuration + t.Run("ClientChecksObjectLockSupport", func(t *testing.T) { + configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{ + Bucket: aws.String(bucketName), + }) + + require.NoError(t, err, "Object Lock configuration check should succeed") + + // S3 clients should see Object Lock is enabled + require.NotNil(t, configResp.ObjectLockConfiguration) + assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled) + t.Log("Object Lock configuration retrieved successfully - S3 clients would see this as supported") + }) + + // Step 3: Client would then configure retention policies and use Object Lock + t.Run("ClientConfiguresRetention", func(t *testing.T) { + // Verify versioning is automatically enabled (required for Object Lock) + versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled") + + // Create an object + key := "protected-backup-object" + content := "Backup data with Object Lock protection" + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: strings.NewReader(content), + }) + require.NoError(t, err) + require.NotNil(t, putResp.VersionId) + + // Set Object Lock retention (what backup clients do to protect data) + retentionUntil := time.Now().Add(24 * time.Hour) + _, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Retention: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + RetainUntilDate: aws.Time(retentionUntil), + }, + }) + require.NoError(t, err) + + // Verify object is protected + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + require.Error(t, err, "Object should be protected by retention policy") + + t.Log("Object Lock retention successfully applied - data is immutable") + }) +} diff --git a/test/s3/retention/s3_retention_test.go b/test/s3/retention/s3_retention_test.go index e3f3222c9..f23f4ddd8 100644 --- a/test/s3/retention/s3_retention_test.go +++ b/test/s3/retention/s3_retention_test.go @@ -1,4 +1,4 @@ -package s3api +package retention import ( "context" diff --git a/test/s3/retention/s3_worm_integration_test.go b/test/s3/retention/s3_worm_integration_test.go index 31217001f..8daa31a46 100644 --- a/test/s3/retention/s3_worm_integration_test.go +++ b/test/s3/retention/s3_worm_integration_test.go @@ -1,4 +1,4 @@ -package s3api +package retention import ( "context" diff --git a/weed/s3api/s3_constants/extend_key.go b/weed/s3api/s3_constants/extend_key.go index e7eee0cc1..edfa4fe1d 100644 --- a/weed/s3api/s3_constants/extend_key.go +++ b/weed/s3api/s3_constants/extend_key.go @@ -35,4 +35,8 @@ const ( // Object lock enabled status ObjectLockEnabled = "Enabled" + + // Bucket versioning status + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" ) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 48e4609e0..e0db5ef9a 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -51,6 +51,9 @@ const ( AmzAclReadAcp = "X-Amz-Grant-Read-Acp" AmzAclWriteAcp = "X-Amz-Grant-Write-Acp" + // S3 Object Lock headers + AmzBucketObjectLockEnabled = "X-Amz-Bucket-Object-Lock-Enabled" + // S3 conditional copy headers AmzCopySourceIfMatch = "X-Amz-Copy-Source-If-Match" AmzCopySourceIfNoneMatch = "X-Amz-Copy-Source-If-None-Match" diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index ecc6af2ac..e9dc10cff 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -136,6 +136,48 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } + + // Check for x-amz-bucket-object-lock-enabled header (S3 standard compliance) + if objectLockHeaderValue := r.Header.Get(s3_constants.AmzBucketObjectLockEnabled); strings.EqualFold(objectLockHeaderValue, "true") { + glog.V(3).Infof("PutBucketHandler: enabling Object Lock and Versioning for bucket %s due to x-amz-bucket-object-lock-enabled header", bucket) + + // Atomically update the configuration of the specified bucket. See the updateBucketConfig + // function definition for detailed documentation on parameters and behavior. + errCode := s3a.updateBucketConfig(bucket, func(bucketConfig *BucketConfig) error { + // Enable versioning (required for Object Lock) + bucketConfig.Versioning = s3_constants.VersioningEnabled + + // Enable Object Lock configuration + if bucketConfig.Entry.Extended == nil { + bucketConfig.Entry.Extended = make(map[string][]byte) + } + + // Create basic Object Lock configuration (enabled without default retention) + // The ObjectLockConfiguration struct is defined below in this file. + objectLockConfig := &ObjectLockConfiguration{ + ObjectLockEnabled: s3_constants.ObjectLockEnabled, + } + + // Store the configuration as XML in extended attributes + configXML, err := xml.Marshal(objectLockConfig) + if err != nil { + return fmt.Errorf("failed to marshal Object Lock configuration to XML: %v", err) + } + + bucketConfig.Entry.Extended[s3_constants.ExtObjectLockConfigKey] = configXML + bucketConfig.Entry.Extended[s3_constants.ExtObjectLockEnabledKey] = []byte(s3_constants.ObjectLockEnabled) + + return nil + }) + + if errCode != s3err.ErrNone { + glog.Errorf("PutBucketHandler: failed to enable Object Lock for bucket %s: %v", bucket, errCode) + s3err.WriteErrorResponse(w, r, errCode) + return + } + glog.V(3).Infof("PutBucketHandler: enabled Object Lock and Versioning for bucket %s", bucket) + } + w.Header().Set("Location", "/"+bucket) writeSuccessResponseEmpty(w, r) }