From 5f77f8733511d6d2bc4f47df7c477c4f9c25a727 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 20 Nov 2025 11:42:22 -0800 Subject: [PATCH] S3: S3 Object Retention API to include XML namespace support (#7517) * Refactor S3 Object Retention API to include XML namespace support and improve compatibility with Veeam. Updated XML tags to remove hardcoded namespaces and added test cases for retention and legal hold configurations without namespaces. * Added XMLNS field setting in both places --- weed/s3api/s3_constants/header.go | 5 ++ ...3api_bucket_handlers_object_lock_config.go | 6 ++ weed/s3api/s3api_object_retention.go | 39 ++++++---- weed/s3api/s3api_object_retention_test.go | 76 +++++++++++++++++++ 4 files changed, 110 insertions(+), 16 deletions(-) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index e4c0ad77b..1ef6f62c5 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -23,6 +23,11 @@ import ( "github.com/gorilla/mux" ) +// S3 XML namespace +const ( + S3Namespace = "http://s3.amazonaws.com/doc/2006-03-01/" +) + // Standard S3 HTTP request constants const ( // S3 storage class diff --git a/weed/s3api/s3api_bucket_handlers_object_lock_config.go b/weed/s3api/s3api_bucket_handlers_object_lock_config.go index c779f80d7..23b52648e 100644 --- a/weed/s3api/s3api_bucket_handlers_object_lock_config.go +++ b/weed/s3api/s3api_bucket_handlers_object_lock_config.go @@ -86,6 +86,9 @@ func (s3a *S3ApiServer) GetObjectLockConfigurationHandler(w http.ResponseWriter, // Check if we have cached Object Lock configuration if bucketConfig.ObjectLockConfig != nil { + // Set namespace for S3 compatibility + bucketConfig.ObjectLockConfig.XMLNS = s3_constants.S3Namespace + // Use cached configuration and marshal it to XML for response marshaledXML, err := xml.Marshal(bucketConfig.ObjectLockConfig) if err != nil { @@ -139,6 +142,9 @@ func (s3a *S3ApiServer) GetObjectLockConfigurationHandler(w http.ResponseWriter, // not just ObjectLockConfig, before resetting the TTL s3a.updateBucketConfigCacheFromEntry(freshEntry) + // Set namespace for S3 compatibility + objectLockConfig.XMLNS = s3_constants.S3Namespace + // Marshal and return the configuration marshaledXML, err := xml.Marshal(objectLockConfig) if err != nil { diff --git a/weed/s3api/s3api_object_retention.go b/weed/s3api/s3api_object_retention.go index 5bb2faf54..ef298eb43 100644 --- a/weed/s3api/s3api_object_retention.go +++ b/weed/s3api/s3api_object_retention.go @@ -57,37 +57,40 @@ const ( // ObjectRetention represents S3 Object Retention configuration type ObjectRetention struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Retention"` - Mode string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Mode,omitempty"` - RetainUntilDate *time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RetainUntilDate,omitempty"` + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"Retention"` + Mode string `xml:"Mode,omitempty"` + RetainUntilDate *time.Time `xml:"RetainUntilDate,omitempty"` } // ObjectLegalHold represents S3 Object Legal Hold configuration type ObjectLegalHold struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LegalHold"` - Status string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Status,omitempty"` + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"LegalHold"` + Status string `xml:"Status,omitempty"` } // ObjectLockConfiguration represents S3 Object Lock Configuration type ObjectLockConfiguration struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration"` - ObjectLockEnabled string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockEnabled,omitempty"` - Rule *ObjectLockRule `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Rule,omitempty"` + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ObjectLockConfiguration"` + ObjectLockEnabled string `xml:"ObjectLockEnabled,omitempty"` + Rule *ObjectLockRule `xml:"Rule,omitempty"` } // ObjectLockRule represents an Object Lock Rule type ObjectLockRule struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Rule"` - DefaultRetention *DefaultRetention `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DefaultRetention,omitempty"` + XMLName xml.Name `xml:"Rule"` + DefaultRetention *DefaultRetention `xml:"DefaultRetention,omitempty"` } // DefaultRetention represents default retention settings // Implements custom XML unmarshal to track if Days/Years were present in XML type DefaultRetention struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DefaultRetention"` - Mode string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Mode,omitempty"` - Days int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Days,omitempty"` - Years int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Years,omitempty"` + XMLName xml.Name `xml:"DefaultRetention"` + Mode string `xml:"Mode,omitempty"` + Days int `xml:"Days,omitempty"` + Years int `xml:"Years,omitempty"` DaysSet bool `xml:"-"` YearsSet bool `xml:"-"` } @@ -102,8 +105,8 @@ func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) type Alias DefaultRetention aux := &struct { *Alias - Days *int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Days,omitempty"` - Years *int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Years,omitempty"` + Days *int `xml:"Days,omitempty"` + Years *int `xml:"Years,omitempty"` }{Alias: (*Alias)(dr)} if err := d.DecodeElement(aux, &start); err != nil { glog.V(2).Infof("DefaultRetention.UnmarshalXML: decode error: %v", err) @@ -245,6 +248,8 @@ func (s3a *S3ApiServer) getObjectRetention(bucket, object, versionId string) (*O return nil, ErrNoRetentionConfiguration } + // Set namespace for S3 compatibility + retention.XMLNS = s3_constants.S3Namespace return retention, nil } @@ -386,6 +391,8 @@ func (s3a *S3ApiServer) getObjectLegalHold(bucket, object, versionId string) (*O return nil, ErrNoLegalHoldConfiguration } + // Set namespace for S3 compatibility + legalHold.XMLNS = s3_constants.S3Namespace return legalHold, nil } diff --git a/weed/s3api/s3api_object_retention_test.go b/weed/s3api/s3api_object_retention_test.go index 20ccf60d9..34c772acd 100644 --- a/weed/s3api/s3api_object_retention_test.go +++ b/weed/s3api/s3api_object_retention_test.go @@ -201,6 +201,30 @@ func TestParseObjectRetention(t *testing.T) { RetainUntilDate: timePtr(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)), }, }, + { + name: "Valid retention XML without namespace (Veeam compatibility)", + xmlBody: ` + GOVERNANCE + 2024-12-31T23:59:59Z + `, + expectError: false, + expectedResult: &ObjectRetention{ + Mode: "GOVERNANCE", + RetainUntilDate: timePtr(time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC)), + }, + }, + { + name: "Valid compliance retention XML without namespace (Veeam compatibility)", + xmlBody: ` + COMPLIANCE + 2025-01-01T00:00:00Z + `, + expectError: false, + expectedResult: &ObjectRetention{ + Mode: "COMPLIANCE", + RetainUntilDate: timePtr(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)), + }, + }, { name: "Empty XML body", xmlBody: "", @@ -311,6 +335,26 @@ func TestParseObjectLegalHold(t *testing.T) { Status: "OFF", }, }, + { + name: "Valid legal hold ON without namespace", + xmlBody: ` + ON + `, + expectError: false, + expectedResult: &ObjectLegalHold{ + Status: "ON", + }, + }, + { + name: "Valid legal hold OFF without namespace", + xmlBody: ` + OFF + `, + expectError: false, + expectedResult: &ObjectLegalHold{ + Status: "OFF", + }, + }, { name: "Empty XML body", xmlBody: "", @@ -405,6 +449,38 @@ func TestParseObjectLockConfiguration(t *testing.T) { }, }, }, + { + name: "Valid object lock configuration without namespace", + xmlBody: ` + Enabled + `, + expectError: false, + expectedResult: &ObjectLockConfiguration{ + ObjectLockEnabled: "Enabled", + }, + }, + { + name: "Valid object lock configuration with rule without namespace", + xmlBody: ` + Enabled + + + GOVERNANCE + 30 + + + `, + expectError: false, + expectedResult: &ObjectLockConfiguration{ + ObjectLockEnabled: "Enabled", + Rule: &ObjectLockRule{ + DefaultRetention: &DefaultRetention{ + Mode: "GOVERNANCE", + Days: 30, + }, + }, + }, + }, { name: "Empty XML body", xmlBody: "",