S3 Object Lock: ensure x-amz-bucket-object-lock-enabled header (#6990)

* ensure x-amz-bucket-object-lock-enabled header * fix tests * combine 2 metadata changes into one * address comments * Update s3api_bucket_handlers.go * Update weed/s3api/s3api_bucket_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update test/s3/retention/object_lock_reproduce_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update test/s3/retention/object_lock_validation_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update test/s3/retention/s3_bucket_object_lock_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_bucket_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_bucket_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update test/s3/retention/s3_bucket_object_lock_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_bucket_handlers.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * package name --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
3 months ago · dde1cf63c2
9 changed files with 458 additions and 2 deletions
--- a/.gitignore
+++ b/.gitignore
@ -107,3 +107,6 @@ test/s3/cors/weed-test.log
 test/s3/cors/weed-server.pid
 /test/s3/cors/test-volume-data
 test/s3/cors/cors.test
+/test/s3/retention/filerldb2
+test/s3/retention/weed-test.log
+test/s3/retention/weed-server.pid
--- a/test/s3/retention/object_lock_reproduce_test.go
+++ b/test/s3/retention/object_lock_reproduce_test.go
@ -0,0 +1,114 @@
+package retention
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestReproduceObjectLockIssue reproduces the Object Lock header processing issue step by step
+func TestReproduceObjectLockIssue(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
+
+	t.Logf("=== Reproducing Object Lock Header Processing Issue ===")
+	t.Logf("Bucket name: %s", bucketName)
+
+	// Step 1: Create bucket with Object Lock enabled header
+	t.Logf("\n1. Creating bucket with ObjectLockEnabledForBucket=true")
+	t.Logf("   This should send x-amz-bucket-object-lock-enabled: true header")
+
+	createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket:                     aws.String(bucketName),
+		ObjectLockEnabledForBucket: true, // This sets the x-amz-bucket-object-lock-enabled header
+	})
+
+	if err != nil {
+		t.Fatalf("Bucket creation failed: %v", err)
+	}
+	t.Logf("✅ Bucket created successfully")
+	t.Logf("   Response: %+v", createResp)
+
+	// Step 2: Check if Object Lock is actually enabled
+	t.Logf("\n2. Checking Object Lock configuration to verify it was enabled")
+
+	objectLockResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+		Bucket: aws.String(bucketName),
+	})
+
+	if err != nil {
+		t.Logf("❌ GetObjectLockConfiguration FAILED: %v", err)
+		t.Logf("   This demonstrates the issue with header processing!")
+		t.Logf("   S3 clients expect this call to succeed if Object Lock is supported")
+		t.Logf("   When this fails, clients conclude that Object Lock is not supported")
+
+		// This failure demonstrates the bug - the bucket was created but Object Lock wasn't enabled
+		t.Logf("\n🐛 BUG CONFIRMED:")
+		t.Logf("   - Bucket creation with ObjectLockEnabledForBucket=true succeeded")
+		t.Logf("   - But GetObjectLockConfiguration fails")
+		t.Logf("   - This means the x-amz-bucket-object-lock-enabled header was ignored")
+
+	} else {
+		t.Logf("✅ GetObjectLockConfiguration succeeded!")
+		t.Logf("   Response: %+v", objectLockResp)
+		t.Logf("   Object Lock is properly enabled - this is the expected behavior")
+	}
+
+	// Step 3: Check versioning status (required for Object Lock)
+	t.Logf("\n3. Checking bucket versioning status (required for Object Lock)")
+
+	versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	t.Logf("   Versioning status: %v", versioningResp.Status)
+	if versioningResp.Status != "Enabled" {
+		t.Logf("   ⚠️  Versioning should be automatically enabled when Object Lock is enabled")
+	}
+
+	// Cleanup
+	t.Logf("\n4. Cleaning up test bucket")
+	_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("   Warning: Failed to delete bucket: %v", err)
+	}
+
+	t.Logf("\n=== Issue Reproduction Complete ===")
+	t.Logf("Expected behavior after fix:")
+	t.Logf("  - CreateBucket with ObjectLockEnabledForBucket=true should enable Object Lock")
+	t.Logf("  - GetObjectLockConfiguration should return enabled configuration")
+	t.Logf("  - Versioning should be automatically enabled")
+}
+
+// TestNormalBucketCreationStillWorks tests that normal bucket creation still works
+func TestNormalBucketCreationStillWorks(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := fmt.Sprintf("normal-test-%d", time.Now().UnixNano())
+
+	t.Logf("=== Testing Normal Bucket Creation ===")
+
+	// Create bucket without Object Lock
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	t.Logf("✅ Normal bucket creation works")
+
+	// Object Lock should NOT be enabled
+	_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.Error(t, err, "GetObjectLockConfiguration should fail for bucket without Object Lock")
+	t.Logf("✅ GetObjectLockConfiguration correctly fails for normal bucket")
+
+	// Cleanup
+	client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+}
--- a/test/s3/retention/object_lock_validation_test.go
+++ b/test/s3/retention/object_lock_validation_test.go
@ -0,0 +1,105 @@
+package retention
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestObjectLockValidation tests that S3 Object Lock functionality works end-to-end
+// This test focuses on the complete Object Lock workflow that S3 clients expect
+func TestObjectLockValidation(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
+
+	t.Logf("=== Validating S3 Object Lock Functionality ===")
+	t.Logf("Bucket: %s", bucketName)
+
+	// Step 1: Create bucket with Object Lock header
+	t.Log("\n1. Creating bucket with x-amz-bucket-object-lock-enabled: true")
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket:                     aws.String(bucketName),
+		ObjectLockEnabledForBucket: true, // This sends x-amz-bucket-object-lock-enabled: true
+	})
+	require.NoError(t, err, "Bucket creation should succeed")
+	defer client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+	t.Log("   ✅ Bucket created successfully")
+
+	// Step 2: Check if Object Lock is supported (standard S3 client behavior)
+	t.Log("\n2. Testing Object Lock support detection")
+	_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "GetObjectLockConfiguration should succeed for Object Lock enabled bucket")
+	t.Log("   ✅ GetObjectLockConfiguration succeeded - Object Lock is properly enabled")
+
+	// Step 3: Verify versioning is enabled (required for Object Lock)
+	t.Log("\n3. Verifying versioning is automatically enabled")
+	versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
+	t.Log("   ✅ Versioning automatically enabled")
+
+	// Step 4: Test actual Object Lock functionality
+	t.Log("\n4. Testing Object Lock retention functionality")
+
+	// Create an object
+	key := "protected-object.dat"
+	content := "Important data that needs immutable protection"
+	putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+		Body:   strings.NewReader(content),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp.VersionId, "Object should have a version ID")
+	t.Log("   ✅ Object created with versioning")
+
+	// Apply Object Lock retention
+	retentionUntil := time.Now().Add(24 * time.Hour)
+	_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+		Retention: &types.ObjectLockRetention{
+			Mode:            types.ObjectLockRetentionModeCompliance,
+			RetainUntilDate: aws.Time(retentionUntil),
+		},
+	})
+	require.NoError(t, err, "Setting Object Lock retention should succeed")
+	t.Log("   ✅ Object Lock retention applied successfully")
+
+	// Verify retention is in effect
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+	})
+	require.Error(t, err, "Object should be protected by retention and cannot be deleted")
+	t.Log("   ✅ Object is properly protected by retention policy")
+
+	// Verify we can read the object (should still work)
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "Reading protected object should still work")
+	defer getResp.Body.Close()
+	t.Log("   ✅ Protected object can still be read")
+
+	t.Log("\n🎉 S3 OBJECT LOCK VALIDATION SUCCESSFUL!")
+	t.Log("   - Bucket creation with Object Lock header works")
+	t.Log("   - Object Lock support detection works (GetObjectLockConfiguration succeeds)")
+	t.Log("   - Versioning is automatically enabled")
+	t.Log("   - Object Lock retention functionality works")
+	t.Log("   - Objects are properly protected from deletion")
+	t.Log("")
+	t.Log("✅ S3 clients will now recognize SeaweedFS as supporting Object Lock!")
+}
--- a/test/s3/retention/s3_bucket_object_lock_test.go
+++ b/test/s3/retention/s3_bucket_object_lock_test.go
@ -0,0 +1,185 @@
+package retention
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestBucketCreationWithObjectLockEnabled tests creating a bucket with the
+// x-amz-bucket-object-lock-enabled header, which is required for S3 Object Lock compatibility
+func TestBucketCreationWithObjectLockEnabled(t *testing.T) {
+	// This test verifies that bucket creation with
+	// x-amz-bucket-object-lock-enabled header should automatically enable Object Lock
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	defer func() {
+		// Best effort cleanup
+		deleteBucket(t, client, bucketName)
+	}()
+
+	// Test 1: Create bucket with Object Lock enabled header using custom HTTP client
+	t.Run("CreateBucketWithObjectLockHeader", func(t *testing.T) {
+		// Create bucket with x-amz-bucket-object-lock-enabled header
+		// This simulates what S3 clients do when testing Object Lock support
+		createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+			Bucket:                     aws.String(bucketName),
+			ObjectLockEnabledForBucket: true, // This should set x-amz-bucket-object-lock-enabled header
+		})
+		require.NoError(t, err)
+		require.NotNil(t, createResp)
+
+		// Verify bucket was created
+		_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+	})
+
+	// Test 2: Verify that Object Lock is automatically enabled for the bucket
+	t.Run("VerifyObjectLockAutoEnabled", func(t *testing.T) {
+		// Try to get the Object Lock configuration
+		// If the header was processed correctly, this should return an enabled configuration
+		configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+			Bucket: aws.String(bucketName),
+		})
+
+		require.NoError(t, err, "GetObjectLockConfiguration should not fail if Object Lock is enabled")
+		require.NotNil(t, configResp.ObjectLockConfiguration, "ObjectLockConfiguration should not be nil")
+		assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled, "Object Lock should be enabled")
+	})
+
+	// Test 3: Verify versioning is automatically enabled (required for Object Lock)
+	t.Run("VerifyVersioningAutoEnabled", func(t *testing.T) {
+		// Object Lock requires versioning to be enabled
+		// When Object Lock is enabled via header, versioning should also be enabled automatically
+		versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+
+		// Versioning should be automatically enabled for Object Lock
+		assert.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled for Object Lock")
+	})
+}
+
+// TestBucketCreationWithoutObjectLockHeader tests normal bucket creation
+// to ensure we don't break existing functionality
+func TestBucketCreationWithoutObjectLockHeader(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	defer deleteBucket(t, client, bucketName)
+
+	// Create bucket without Object Lock header
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	// Verify bucket was created
+	_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	// Object Lock should NOT be enabled
+	_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+		Bucket: aws.String(bucketName),
+	})
+	// This should fail since Object Lock is not enabled
+	require.Error(t, err)
+	t.Logf("GetObjectLockConfiguration correctly failed for bucket without Object Lock: %v", err)
+
+	// Versioning should not be enabled by default
+	versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	// Should be either empty/unset or Suspended, but not Enabled
+	if versioningResp.Status != types.BucketVersioningStatusEnabled {
+		t.Logf("Versioning correctly not enabled: %v", versioningResp.Status)
+	} else {
+		t.Errorf("Versioning should not be enabled for bucket without Object Lock header")
+	}
+}
+
+// TestS3ObjectLockWorkflow tests the complete Object Lock workflow that S3 clients would use
+func TestS3ObjectLockWorkflow(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	defer deleteBucket(t, client, bucketName)
+
+	// Step 1: Client creates bucket with Object Lock enabled
+	t.Run("ClientCreatesBucket", func(t *testing.T) {
+		_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+			Bucket:                     aws.String(bucketName),
+			ObjectLockEnabledForBucket: true,
+		})
+		require.NoError(t, err)
+	})
+
+	// Step 2: Client checks if Object Lock is supported by getting the configuration
+	t.Run("ClientChecksObjectLockSupport", func(t *testing.T) {
+		configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
+			Bucket: aws.String(bucketName),
+		})
+
+		require.NoError(t, err, "Object Lock configuration check should succeed")
+
+		// S3 clients should see Object Lock is enabled
+		require.NotNil(t, configResp.ObjectLockConfiguration)
+		assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled)
+		t.Log("Object Lock configuration retrieved successfully - S3 clients would see this as supported")
+	})
+
+	// Step 3: Client would then configure retention policies and use Object Lock
+	t.Run("ClientConfiguresRetention", func(t *testing.T) {
+		// Verify versioning is automatically enabled (required for Object Lock)
+		versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+		require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
+
+		// Create an object
+		key := "protected-backup-object"
+		content := "Backup data with Object Lock protection"
+		putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(content),
+		})
+		require.NoError(t, err)
+		require.NotNil(t, putResp.VersionId)
+
+		// Set Object Lock retention (what backup clients do to protect data)
+		retentionUntil := time.Now().Add(24 * time.Hour)
+		_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Retention: &types.ObjectLockRetention{
+				Mode:            types.ObjectLockRetentionModeCompliance,
+				RetainUntilDate: aws.Time(retentionUntil),
+			},
+		})
+		require.NoError(t, err)
+
+		// Verify object is protected
+		_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+		})
+		require.Error(t, err, "Object should be protected by retention policy")
+
+		t.Log("Object Lock retention successfully applied - data is immutable")
+	})
+}
--- a/test/s3/retention/s3_retention_test.go
+++ b/test/s3/retention/s3_retention_test.go
@ -1,4 +1,4 @@
-package s3api
+package retention

 import (
 	"context"
--- a/test/s3/retention/s3_worm_integration_test.go
+++ b/test/s3/retention/s3_worm_integration_test.go
@ -1,4 +1,4 @@
-package s3api
+package retention

 import (
 	"context"
--- a/weed/s3api/s3_constants/extend_key.go
+++ b/weed/s3api/s3_constants/extend_key.go
@ -35,4 +35,8 @@ const (

 	// Object lock enabled status
 	ObjectLockEnabled = "Enabled"
+
+	// Bucket versioning status
+	VersioningEnabled   = "Enabled"
+	VersioningSuspended = "Suspended"
 )
--- a/weed/s3api/s3_constants/header.go
+++ b/weed/s3api/s3_constants/header.go
@ -51,6 +51,9 @@ const (
 	AmzAclReadAcp     = "X-Amz-Grant-Read-Acp"
 	AmzAclWriteAcp    = "X-Amz-Grant-Write-Acp"

+	// S3 Object Lock headers
+	AmzBucketObjectLockEnabled = "X-Amz-Bucket-Object-Lock-Enabled"
+
 	// S3 conditional copy headers
 	AmzCopySourceIfMatch           = "X-Amz-Copy-Source-If-Match"
 	AmzCopySourceIfNoneMatch       = "X-Amz-Copy-Source-If-None-Match"
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@ -136,6 +136,48 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
+
+	// Check for x-amz-bucket-object-lock-enabled header (S3 standard compliance)
+	if objectLockHeaderValue := r.Header.Get(s3_constants.AmzBucketObjectLockEnabled); strings.EqualFold(objectLockHeaderValue, "true") {
+		glog.V(3).Infof("PutBucketHandler: enabling Object Lock and Versioning for bucket %s due to x-amz-bucket-object-lock-enabled header", bucket)
+
+		// Atomically update the configuration of the specified bucket. See the updateBucketConfig
+		// function definition for detailed documentation on parameters and behavior.
+		errCode := s3a.updateBucketConfig(bucket, func(bucketConfig *BucketConfig) error {
+			// Enable versioning (required for Object Lock)
+			bucketConfig.Versioning = s3_constants.VersioningEnabled
+
+			// Enable Object Lock configuration
+			if bucketConfig.Entry.Extended == nil {
+				bucketConfig.Entry.Extended = make(map[string][]byte)
+			}
+
+			// Create basic Object Lock configuration (enabled without default retention)
+			// The ObjectLockConfiguration struct is defined below in this file.
+			objectLockConfig := &ObjectLockConfiguration{
+				ObjectLockEnabled: s3_constants.ObjectLockEnabled,
+			}
+
+			// Store the configuration as XML in extended attributes
+			configXML, err := xml.Marshal(objectLockConfig)
+			if err != nil {
+				return fmt.Errorf("failed to marshal Object Lock configuration to XML: %v", err)
+			}
+
+			bucketConfig.Entry.Extended[s3_constants.ExtObjectLockConfigKey] = configXML
+			bucketConfig.Entry.Extended[s3_constants.ExtObjectLockEnabledKey] = []byte(s3_constants.ObjectLockEnabled)
+
+			return nil
+		})
+
+		if errCode != s3err.ErrNone {
+			glog.Errorf("PutBucketHandler: failed to enable Object Lock for bucket %s: %v", bucket, errCode)
+			s3err.WriteErrorResponse(w, r, errCode)
+			return
+		}
+		glog.V(3).Infof("PutBucketHandler: enabled Object Lock and Versioning for bucket %s", bucket)
+	}
+
 	w.Header().Set("Location", "/"+bucket)
 	writeSuccessResponseEmpty(w, r)
 }