From 54dd4f091d4b373c1ef606dfebf028eb90aa359a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 28 Mar 2026 11:10:31 -0700 Subject: [PATCH] s3lifecycle: add lifecycle rule evaluator package and extend XML types (#8807) * s3api: extend lifecycle XML types with NoncurrentVersionExpiration, AbortIncompleteMultipartUpload Add missing S3 lifecycle rule types to the XML data model: - NoncurrentVersionExpiration with NoncurrentDays and NewerNoncurrentVersions - NoncurrentVersionTransition with NoncurrentDays and StorageClass - AbortIncompleteMultipartUpload with DaysAfterInitiation - Filter.ObjectSizeGreaterThan and ObjectSizeLessThan - And.ObjectSizeGreaterThan and ObjectSizeLessThan - Filter.UnmarshalXML to properly parse Tag, And, and size filter elements Each new type follows the existing set-field pattern for conditional XML marshaling. No behavior changes - these types are not yet wired into handlers or the lifecycle worker. * s3lifecycle: add lifecycle rule evaluator package New package weed/s3api/s3lifecycle/ provides a pure-function lifecycle rule evaluation engine. The evaluator accepts flattened Rule structs and ObjectInfo metadata, and returns the appropriate Action. Components: - evaluator.go: Evaluate() for per-object actions with S3 priority ordering (delete marker > noncurrent version > current expiration), ShouldExpireNoncurrentVersion() with NewerNoncurrentVersions support, EvaluateMPUAbort() for multipart upload rules - filter.go: prefix, tag, and size-based filter matching - tags.go: ExtractTags() extracts S3 tags from filer Extended metadata, HasTagRules() for scan-time optimization - version_time.go: GetVersionTimestamp() extracts timestamps from SeaweedFS version IDs (both old and new format) Comprehensive test coverage: 54 tests covering all action types, filter combinations, edge cases, and version ID formats. * s3api: add UnmarshalXML for Expiration, Transition, ExpireDeleteMarker Add UnmarshalXML methods that set the internal 'set' flag during XML parsing. Previously these flags were only set programmatically, causing XML round-trip to drop elements. This ensures lifecycle configurations stored as XML survive unmarshal/marshal cycles correctly. Add comprehensive XML round-trip tests for all lifecycle rule types including NoncurrentVersionExpiration, AbortIncompleteMultipartUpload, Filter with Tag/And/size constraints, and a complete Terraform-style lifecycle configuration. * s3lifecycle: address review feedback - Fix version_time.go overflow: guard timestampPart > MaxInt64 before the inversion subtraction to prevent uint64 wrap - Make all expiry checks inclusive (!now.Before instead of now.After) so actions trigger at the exact scheduled instant - Add NoncurrentIndex to ObjectInfo so Evaluate() can properly handle NewerNoncurrentVersions via ShouldExpireNoncurrentVersion() - Add test for high-bit overflow version ID * s3lifecycle: guard ShouldExpireNoncurrentVersion against zero SuccessorModTime Add early return when obj.IsLatest or obj.SuccessorModTime.IsZero() to prevent premature expiration of versions with uninitialized successor timestamps (zero value would compute to epoch, always expired). --------- Co-authored-by: Copilot --- weed/s3api/s3api_policy.go | 213 ++++++++- weed/s3api/s3api_policy_test.go | 231 +++++++++ weed/s3api/s3lifecycle/evaluator.go | 127 +++++ weed/s3api/s3lifecycle/evaluator_test.go | 495 ++++++++++++++++++++ weed/s3api/s3lifecycle/filter.go | 56 +++ weed/s3api/s3lifecycle/filter_test.go | 79 ++++ weed/s3api/s3lifecycle/rule.go | 95 ++++ weed/s3api/s3lifecycle/tags.go | 34 ++ weed/s3api/s3lifecycle/tags_test.go | 89 ++++ weed/s3api/s3lifecycle/version_time.go | 42 ++ weed/s3api/s3lifecycle/version_time_test.go | 74 +++ 11 files changed, 1523 insertions(+), 12 deletions(-) create mode 100644 weed/s3api/s3api_policy_test.go create mode 100644 weed/s3api/s3lifecycle/evaluator.go create mode 100644 weed/s3api/s3lifecycle/evaluator_test.go create mode 100644 weed/s3api/s3lifecycle/filter.go create mode 100644 weed/s3api/s3lifecycle/filter_test.go create mode 100644 weed/s3api/s3lifecycle/rule.go create mode 100644 weed/s3api/s3lifecycle/tags.go create mode 100644 weed/s3api/s3lifecycle/tags_test.go create mode 100644 weed/s3api/s3lifecycle/version_time.go create mode 100644 weed/s3api/s3lifecycle/version_time_test.go diff --git a/weed/s3api/s3api_policy.go b/weed/s3api/s3api_policy.go index dab2e3f02..cb715cba9 100644 --- a/weed/s3api/s3api_policy.go +++ b/weed/s3api/s3api_policy.go @@ -22,13 +22,16 @@ type Lifecycle struct { // Rule - a rule for lifecycle configuration. type Rule struct { - XMLName xml.Name `xml:"Rule"` - ID string `xml:"ID,omitempty"` - Status ruleStatus `xml:"Status"` - Filter Filter `xml:"Filter,omitempty"` - Prefix Prefix `xml:"Prefix,omitempty"` - Expiration Expiration `xml:"Expiration,omitempty"` - Transition Transition `xml:"Transition,omitempty"` + XMLName xml.Name `xml:"Rule"` + ID string `xml:"ID,omitempty"` + Status ruleStatus `xml:"Status"` + Filter Filter `xml:"Filter,omitempty"` + Prefix Prefix `xml:"Prefix,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + Transition Transition `xml:"Transition,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` + AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"` } // Filter - a filter for a lifecycle configuration Rule. @@ -43,6 +46,9 @@ type Filter struct { Tag Tag tagSet bool + + ObjectSizeGreaterThan int64 + ObjectSizeLessThan int64 } // Prefix holds the prefix xml tag in and @@ -80,17 +86,83 @@ func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } - if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { - return err + if f.andSet { + if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { + return err + } + } else if f.tagSet { + if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { + return err + } + } else { + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + } + if f.ObjectSizeGreaterThan > 0 { + if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil { + return err + } + } + if f.ObjectSizeLessThan > 0 { + if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil { + return err + } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } +// UnmarshalXML decodes Filter from XML, handling all child elements. +func (f *Filter) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + f.set = true + for { + tok, err := d.Token() + if err != nil { + return err + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "Prefix": + if err := d.DecodeElement(&f.Prefix, &t); err != nil { + return err + } + case "Tag": + f.tagSet = true + if err := d.DecodeElement(&f.Tag, &t); err != nil { + return err + } + case "And": + f.andSet = true + if err := d.DecodeElement(&f.And, &t); err != nil { + return err + } + case "ObjectSizeGreaterThan": + if err := d.DecodeElement(&f.ObjectSizeGreaterThan, &t); err != nil { + return err + } + case "ObjectSizeLessThan": + if err := d.DecodeElement(&f.ObjectSizeLessThan, &t); err != nil { + return err + } + default: + if err := d.Skip(); err != nil { + return err + } + } + case xml.EndElement: + return nil + } + } +} + // And - a tag to combine a prefix and multiple tags for lifecycle configuration rule. type And struct { - XMLName xml.Name `xml:"And"` - Prefix Prefix `xml:"Prefix,omitempty"` - Tags []Tag `xml:"Tag,omitempty"` + XMLName xml.Name `xml:"And"` + Prefix Prefix `xml:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty"` + ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"` } // Expiration - expiration actions for a rule in lifecycle configuration. @@ -112,6 +184,17 @@ func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement) return enc.EncodeElement(expirationWrapper(e), startElement) } +func (e *Expiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type wrapper Expiration + var w wrapper + if err := d.DecodeElement(&w, &start); err != nil { + return err + } + *e = Expiration(w) + e.set = true + return nil +} + // ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. type ExpireDeleteMarker struct { val bool @@ -126,6 +209,15 @@ func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartEle return e.EncodeElement(b.val, startElement) } +func (b *ExpireDeleteMarker) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var v bool + if err := d.DecodeElement(&v, &start); err != nil { + return err + } + *b = ExpireDeleteMarker{val: v, set: true} + return nil +} + // ExpirationDate is a embedded type containing time.Time to unmarshal // Date in Expiration type ExpirationDate struct { @@ -160,5 +252,102 @@ func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { return enc.EncodeElement(transitionWrapper(t), start) } +func (t *Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type wrapper Transition + var w wrapper + if err := d.DecodeElement(&w, &start); err != nil { + return err + } + *t = Transition(w) + t.set = true + return nil +} + // TransitionDays is a type alias to unmarshal Days in Transition type TransitionDays int + +// NoncurrentVersionExpiration - expiration actions for non-current object versions. +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays int `xml:"NoncurrentDays,omitempty"` + NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"` + + set bool +} + +// MarshalXML encodes NoncurrentVersionExpiration field into an XML form. +func (n NoncurrentVersionExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !n.set { + return nil + } + type wrapper NoncurrentVersionExpiration + return enc.EncodeElement(wrapper(n), start) +} + +func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type wrapper NoncurrentVersionExpiration + var w wrapper + if err := d.DecodeElement(&w, &start); err != nil { + return err + } + *n = NoncurrentVersionExpiration(w) + n.set = true + return nil +} + +// NoncurrentVersionTransition - transition actions for non-current object versions. +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition"` + NoncurrentDays int `xml:"NoncurrentDays,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + + set bool +} + +// MarshalXML encodes NoncurrentVersionTransition field into an XML form. +func (n NoncurrentVersionTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !n.set { + return nil + } + type wrapper NoncurrentVersionTransition + return enc.EncodeElement(wrapper(n), start) +} + +func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type wrapper NoncurrentVersionTransition + var w wrapper + if err := d.DecodeElement(&w, &start); err != nil { + return err + } + *n = NoncurrentVersionTransition(w) + n.set = true + return nil +} + +// AbortIncompleteMultipartUpload - abort action for incomplete multipart uploads. +type AbortIncompleteMultipartUpload struct { + XMLName xml.Name `xml:"AbortIncompleteMultipartUpload"` + DaysAfterInitiation int `xml:"DaysAfterInitiation,omitempty"` + + set bool +} + +// MarshalXML encodes AbortIncompleteMultipartUpload field into an XML form. +func (a AbortIncompleteMultipartUpload) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !a.set { + return nil + } + type wrapper AbortIncompleteMultipartUpload + return enc.EncodeElement(wrapper(a), start) +} + +func (a *AbortIncompleteMultipartUpload) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type wrapper AbortIncompleteMultipartUpload + var w wrapper + if err := d.DecodeElement(&w, &start); err != nil { + return err + } + *a = AbortIncompleteMultipartUpload(w) + a.set = true + return nil +} diff --git a/weed/s3api/s3api_policy_test.go b/weed/s3api/s3api_policy_test.go new file mode 100644 index 000000000..b14b4f824 --- /dev/null +++ b/weed/s3api/s3api_policy_test.go @@ -0,0 +1,231 @@ +package s3api + +import ( + "encoding/xml" + "strings" + "testing" +) + +func TestLifecycleXMLRoundTrip_NoncurrentVersionExpiration(t *testing.T) { + input := ` + + expire-noncurrent + Enabled + + + 30 + 2 + + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if len(lc.Rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(lc.Rules)) + } + rule := lc.Rules[0] + if rule.ID != "expire-noncurrent" { + t.Errorf("expected ID 'expire-noncurrent', got %q", rule.ID) + } + if rule.NoncurrentVersionExpiration.NoncurrentDays != 30 { + t.Errorf("expected NoncurrentDays=30, got %d", rule.NoncurrentVersionExpiration.NoncurrentDays) + } + if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions != 2 { + t.Errorf("expected NewerNoncurrentVersions=2, got %d", rule.NoncurrentVersionExpiration.NewerNoncurrentVersions) + } + + // Re-marshal and verify it round-trips. + out, err := xml.Marshal(lc) + if err != nil { + t.Fatalf("marshal: %v", err) + } + s := string(out) + if !strings.Contains(s, "30") { + t.Errorf("marshaled XML missing NoncurrentDays: %s", s) + } + if !strings.Contains(s, "2") { + t.Errorf("marshaled XML missing NewerNoncurrentVersions: %s", s) + } +} + +func TestLifecycleXMLRoundTrip_AbortIncompleteMultipartUpload(t *testing.T) { + input := ` + + abort-mpu + Enabled + + + 7 + + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + rule := lc.Rules[0] + if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != 7 { + t.Errorf("expected DaysAfterInitiation=7, got %d", rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) + } + + out, err := xml.Marshal(lc) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(out), "7") { + t.Errorf("marshaled XML missing DaysAfterInitiation: %s", string(out)) + } +} + +func TestLifecycleXMLRoundTrip_FilterWithTag(t *testing.T) { + input := ` + + tag-filter + Enabled + + envdev + + 7 + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + rule := lc.Rules[0] + if !rule.Filter.tagSet { + t.Error("expected Filter.tagSet to be true") + } + if rule.Filter.Tag.Key != "env" || rule.Filter.Tag.Value != "dev" { + t.Errorf("expected Tag{env:dev}, got Tag{%s:%s}", rule.Filter.Tag.Key, rule.Filter.Tag.Value) + } +} + +func TestLifecycleXMLRoundTrip_FilterWithAnd(t *testing.T) { + input := ` + + and-filter + Enabled + + + logs/ + envdev + tierhot + 1024 + 1048576 + + + 7 + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + rule := lc.Rules[0] + if !rule.Filter.andSet { + t.Error("expected Filter.andSet to be true") + } + if rule.Filter.And.Prefix.String() != "logs/" { + t.Errorf("expected And.Prefix='logs/', got %q", rule.Filter.And.Prefix.String()) + } + if len(rule.Filter.And.Tags) != 2 { + t.Fatalf("expected 2 And tags, got %d", len(rule.Filter.And.Tags)) + } + if rule.Filter.And.ObjectSizeGreaterThan != 1024 { + t.Errorf("expected ObjectSizeGreaterThan=1024, got %d", rule.Filter.And.ObjectSizeGreaterThan) + } + if rule.Filter.And.ObjectSizeLessThan != 1048576 { + t.Errorf("expected ObjectSizeLessThan=1048576, got %d", rule.Filter.And.ObjectSizeLessThan) + } +} + +func TestLifecycleXMLRoundTrip_FilterWithSizeOnly(t *testing.T) { + input := ` + + size-filter + Enabled + + 512 + 10485760 + + 30 + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + rule := lc.Rules[0] + if rule.Filter.ObjectSizeGreaterThan != 512 { + t.Errorf("expected ObjectSizeGreaterThan=512, got %d", rule.Filter.ObjectSizeGreaterThan) + } + if rule.Filter.ObjectSizeLessThan != 10485760 { + t.Errorf("expected ObjectSizeLessThan=10485760, got %d", rule.Filter.ObjectSizeLessThan) + } +} + +func TestLifecycleXMLRoundTrip_CompleteRule(t *testing.T) { + // A complete lifecycle config similar to what Terraform sends. + input := ` + + rotation + + Enabled + 30 + + 1 + + + 1 + + +` + + var lc Lifecycle + if err := xml.Unmarshal([]byte(input), &lc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + rule := lc.Rules[0] + if rule.ID != "rotation" { + t.Errorf("expected ID 'rotation', got %q", rule.ID) + } + if rule.Expiration.Days != 30 { + t.Errorf("expected Expiration.Days=30, got %d", rule.Expiration.Days) + } + if rule.NoncurrentVersionExpiration.NoncurrentDays != 1 { + t.Errorf("expected NoncurrentDays=1, got %d", rule.NoncurrentVersionExpiration.NoncurrentDays) + } + if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != 1 { + t.Errorf("expected DaysAfterInitiation=1, got %d", rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) + } + + // Re-marshal and verify all fields survive. + out, err := xml.Marshal(lc) + if err != nil { + t.Fatalf("marshal: %v", err) + } + s := string(out) + for _, expected := range []string{ + "30", + "1", + "1", + } { + if !strings.Contains(s, expected) { + t.Errorf("marshaled XML missing %q: %s", expected, s) + } + } +} diff --git a/weed/s3api/s3lifecycle/evaluator.go b/weed/s3api/s3lifecycle/evaluator.go new file mode 100644 index 000000000..415d07615 --- /dev/null +++ b/weed/s3api/s3lifecycle/evaluator.go @@ -0,0 +1,127 @@ +package s3lifecycle + +import "time" + +// Evaluate checks the given lifecycle rules against an object and returns +// the highest-priority action that applies. The evaluation follows S3's +// action priority: +// 1. ExpiredObjectDeleteMarker (delete marker is sole version) +// 2. NoncurrentVersionExpiration (non-current version age/count) +// 3. Current version Expiration (Days or Date) +// +// AbortIncompleteMultipartUpload is evaluated separately since it applies +// to uploads, not objects. Use EvaluateMPUAbort for that. +func Evaluate(rules []Rule, obj ObjectInfo, now time.Time) EvalResult { + // Phase 1: ExpiredObjectDeleteMarker + if obj.IsDeleteMarker && obj.IsLatest && obj.NumVersions == 1 { + for _, rule := range rules { + if rule.Status != "Enabled" { + continue + } + if !matchesFilter(rule, obj) { + continue + } + if rule.ExpiredObjectDeleteMarker { + return EvalResult{Action: ActionExpireDeleteMarker, RuleID: rule.ID} + } + } + } + + // Phase 2: NoncurrentVersionExpiration + if !obj.IsLatest && !obj.SuccessorModTime.IsZero() { + for _, rule := range rules { + if ShouldExpireNoncurrentVersion(rule, obj, obj.NoncurrentIndex, now) { + return EvalResult{Action: ActionDeleteVersion, RuleID: rule.ID} + } + } + } + + // Phase 3: Current version Expiration + if obj.IsLatest && !obj.IsDeleteMarker { + for _, rule := range rules { + if rule.Status != "Enabled" { + continue + } + if !matchesFilter(rule, obj) { + continue + } + // Date-based expiration + if !rule.ExpirationDate.IsZero() && !now.Before(rule.ExpirationDate) { + return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID} + } + // Days-based expiration + if rule.ExpirationDays > 0 { + expiryTime := expectedExpiryTime(obj.ModTime, rule.ExpirationDays) + if !now.Before(expiryTime) { + return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID} + } + } + } + } + + return EvalResult{Action: ActionNone} +} + +// ShouldExpireNoncurrentVersion checks whether a non-current version should +// be expired considering both NoncurrentDays and NewerNoncurrentVersions. +// noncurrentIndex is the 0-based position among non-current versions sorted +// newest-first (0 = newest non-current version). +func ShouldExpireNoncurrentVersion(rule Rule, obj ObjectInfo, noncurrentIndex int, now time.Time) bool { + if rule.Status != "Enabled" { + return false + } + if rule.NoncurrentVersionExpirationDays <= 0 { + return false + } + if obj.IsLatest || obj.SuccessorModTime.IsZero() { + return false + } + if !matchesFilter(rule, obj) { + return false + } + + // Check age threshold. + expiryTime := expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpirationDays) + if now.Before(expiryTime) { + return false + } + + // Check NewerNoncurrentVersions count threshold. + if rule.NewerNoncurrentVersions > 0 && noncurrentIndex < rule.NewerNoncurrentVersions { + return false + } + + return true +} + +// EvaluateMPUAbort finds the applicable AbortIncompleteMultipartUpload rule +// for a multipart upload with the given key prefix and creation time. +func EvaluateMPUAbort(rules []Rule, uploadKey string, createdAt time.Time, now time.Time) EvalResult { + for _, rule := range rules { + if rule.Status != "Enabled" { + continue + } + if rule.AbortMPUDaysAfterInitiation <= 0 { + continue + } + if !matchesPrefix(rule.Prefix, uploadKey) { + continue + } + cutoff := createdAt.Add(time.Duration(rule.AbortMPUDaysAfterInitiation) * 24 * time.Hour) + if !now.Before(cutoff) { + return EvalResult{Action: ActionAbortMultipartUpload, RuleID: rule.ID} + } + } + return EvalResult{Action: ActionNone} +} + +// expectedExpiryTime computes the expiration time given a reference time and +// a number of days. Following S3 semantics, expiration happens at midnight UTC +// of the day after the specified number of days. +func expectedExpiryTime(refTime time.Time, days int) time.Time { + if days == 0 { + return refTime + } + t := refTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour) + return t.Truncate(24 * time.Hour) +} diff --git a/weed/s3api/s3lifecycle/evaluator_test.go b/weed/s3api/s3lifecycle/evaluator_test.go new file mode 100644 index 000000000..aa58e4bc8 --- /dev/null +++ b/weed/s3api/s3lifecycle/evaluator_test.go @@ -0,0 +1,495 @@ +package s3lifecycle + +import ( + "testing" + "time" +) + +var now = time.Date(2026, 3, 27, 12, 0, 0, 0, time.UTC) + +func TestEvaluate_ExpirationDays(t *testing.T) { + rules := []Rule{{ + ID: "expire-30d", Status: "Enabled", + ExpirationDays: 30, + }} + + t.Run("object_older_than_days_is_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/file.txt", IsLatest: true, + ModTime: now.Add(-31 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + assertEqual(t, "expire-30d", result.RuleID) + }) + + t.Run("object_younger_than_days_is_not_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/file.txt", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("non_latest_version_not_affected_by_expiration_days", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/file.txt", IsLatest: false, + ModTime: now.Add(-60 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("delete_marker_not_affected_by_expiration_days", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/file.txt", IsLatest: true, IsDeleteMarker: true, + ModTime: now.Add(-60 * 24 * time.Hour), NumVersions: 3, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_ExpirationDate(t *testing.T) { + expirationDate := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC) + rules := []Rule{{ + ID: "expire-date", Status: "Enabled", + ExpirationDate: expirationDate, + }} + + t.Run("object_expired_after_date", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-60 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + }) + + t.Run("object_not_expired_before_date", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-1 * time.Hour), + } + beforeDate := time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC) + result := Evaluate(rules, obj, beforeDate) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_ExpiredObjectDeleteMarker(t *testing.T) { + rules := []Rule{{ + ID: "cleanup-markers", Status: "Enabled", + ExpiredObjectDeleteMarker: true, + }} + + t.Run("sole_delete_marker_is_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, IsDeleteMarker: true, + NumVersions: 1, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionExpireDeleteMarker, result.Action) + }) + + t.Run("delete_marker_with_other_versions_not_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, IsDeleteMarker: true, + NumVersions: 3, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("non_latest_delete_marker_not_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, IsDeleteMarker: true, + NumVersions: 1, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("non_delete_marker_not_affected", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, IsDeleteMarker: false, + NumVersions: 1, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_NoncurrentVersionExpiration(t *testing.T) { + rules := []Rule{{ + ID: "expire-noncurrent", Status: "Enabled", + NoncurrentVersionExpirationDays: 30, + }} + + t.Run("old_noncurrent_version_is_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-45 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteVersion, result.Action) + }) + + t.Run("recent_noncurrent_version_is_not_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-10 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("latest_version_not_affected", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-60 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestShouldExpireNoncurrentVersion(t *testing.T) { + rule := Rule{ + ID: "noncurrent-rule", Status: "Enabled", + NoncurrentVersionExpirationDays: 30, + NewerNoncurrentVersions: 2, + } + + t.Run("old_version_beyond_count_is_expired", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-45 * 24 * time.Hour), + } + // noncurrentIndex=2 means this is the 3rd noncurrent version (0-indexed) + // With NewerNoncurrentVersions=2, indices 0 and 1 are kept. + if !ShouldExpireNoncurrentVersion(rule, obj, 2, now) { + t.Error("expected version at index 2 to be expired") + } + }) + + t.Run("old_version_within_count_is_kept", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-45 * 24 * time.Hour), + } + // noncurrentIndex=1 is within the keep threshold (NewerNoncurrentVersions=2). + if ShouldExpireNoncurrentVersion(rule, obj, 1, now) { + t.Error("expected version at index 1 to be kept") + } + }) + + t.Run("recent_version_beyond_count_is_kept", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-5 * 24 * time.Hour), + } + // Even at index 5 (beyond count), if too young, it's kept. + if ShouldExpireNoncurrentVersion(rule, obj, 5, now) { + t.Error("expected recent version to be kept regardless of index") + } + }) + + t.Run("disabled_rule_never_expires", func(t *testing.T) { + disabled := Rule{ + ID: "disabled", Status: "Disabled", + NoncurrentVersionExpirationDays: 1, + } + obj := ObjectInfo{ + Key: "file.txt", IsLatest: false, + SuccessorModTime: now.Add(-365 * 24 * time.Hour), + } + if ShouldExpireNoncurrentVersion(disabled, obj, 10, now) { + t.Error("disabled rule should never expire") + } + }) +} + +func TestEvaluate_PrefixFilter(t *testing.T) { + rules := []Rule{{ + ID: "logs-only", Status: "Enabled", + Prefix: "logs/", + ExpirationDays: 7, + }} + + t.Run("matching_prefix", func(t *testing.T) { + obj := ObjectInfo{ + Key: "logs/app.log", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + }) + + t.Run("non_matching_prefix", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/file.txt", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_TagFilter(t *testing.T) { + rules := []Rule{{ + ID: "temp-only", Status: "Enabled", + ExpirationDays: 1, + FilterTags: map[string]string{"env": "temp"}, + }} + + t.Run("matching_tags", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-5 * 24 * time.Hour), + Tags: map[string]string{"env": "temp", "project": "foo"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + }) + + t.Run("missing_tag", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-5 * 24 * time.Hour), + Tags: map[string]string{"project": "foo"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("wrong_tag_value", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-5 * 24 * time.Hour), + Tags: map[string]string{"env": "prod"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("nil_object_tags", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-5 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_SizeFilter(t *testing.T) { + rules := []Rule{{ + ID: "large-files", Status: "Enabled", + ExpirationDays: 7, + FilterSizeGreaterThan: 1024 * 1024, // > 1 MB + FilterSizeLessThan: 100 * 1024 * 1024, // < 100 MB + }} + + t.Run("matching_size", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.bin", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 10 * 1024 * 1024, // 10 MB + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + }) + + t.Run("too_small", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.bin", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 512, // 512 bytes + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("too_large", func(t *testing.T) { + obj := ObjectInfo{ + Key: "file.bin", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 200 * 1024 * 1024, // 200 MB + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_CombinedFilters(t *testing.T) { + rules := []Rule{{ + ID: "combined", Status: "Enabled", + Prefix: "logs/", + ExpirationDays: 7, + FilterTags: map[string]string{"env": "dev"}, + FilterSizeGreaterThan: 100, + }} + + t.Run("all_filters_match", func(t *testing.T) { + obj := ObjectInfo{ + Key: "logs/app.log", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 1024, + Tags: map[string]string{"env": "dev"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + }) + + t.Run("prefix_doesnt_match", func(t *testing.T) { + obj := ObjectInfo{ + Key: "data/app.log", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 1024, + Tags: map[string]string{"env": "dev"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("tag_doesnt_match", func(t *testing.T) { + obj := ObjectInfo{ + Key: "logs/app.log", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 1024, + Tags: map[string]string{"env": "prod"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("size_doesnt_match", func(t *testing.T) { + obj := ObjectInfo{ + Key: "logs/app.log", IsLatest: true, + ModTime: now.Add(-10 * 24 * time.Hour), + Size: 50, // too small + Tags: map[string]string{"env": "dev"}, + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestEvaluate_DisabledRule(t *testing.T) { + rules := []Rule{{ + ID: "disabled", Status: "Disabled", + ExpirationDays: 1, + }} + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, + ModTime: now.Add(-365 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionNone, result.Action) +} + +func TestEvaluate_MultipleRules_Priority(t *testing.T) { + t.Run("delete_marker_takes_priority_over_expiration", func(t *testing.T) { + rules := []Rule{ + {ID: "expire", Status: "Enabled", ExpirationDays: 1}, + {ID: "marker", Status: "Enabled", ExpiredObjectDeleteMarker: true}, + } + obj := ObjectInfo{ + Key: "file.txt", IsLatest: true, IsDeleteMarker: true, + NumVersions: 1, ModTime: now.Add(-10 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionExpireDeleteMarker, result.Action) + assertEqual(t, "marker", result.RuleID) + }) + + t.Run("first_matching_expiration_rule_wins", func(t *testing.T) { + rules := []Rule{ + {ID: "rule1", Status: "Enabled", ExpirationDays: 30, Prefix: "logs/"}, + {ID: "rule2", Status: "Enabled", ExpirationDays: 7}, + } + obj := ObjectInfo{ + Key: "logs/app.log", IsLatest: true, + ModTime: now.Add(-31 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) + assertEqual(t, "rule1", result.RuleID) + }) +} + +func TestEvaluate_EmptyPrefix(t *testing.T) { + rules := []Rule{{ + ID: "all", Status: "Enabled", + ExpirationDays: 30, + }} + obj := ObjectInfo{ + Key: "any/path/file.txt", IsLatest: true, + ModTime: now.Add(-31 * 24 * time.Hour), + } + result := Evaluate(rules, obj, now) + assertAction(t, ActionDeleteObject, result.Action) +} + +func TestEvaluateMPUAbort(t *testing.T) { + rules := []Rule{{ + ID: "abort-mpu", Status: "Enabled", + AbortMPUDaysAfterInitiation: 7, + }} + + t.Run("old_upload_is_aborted", func(t *testing.T) { + result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-10*24*time.Hour), now) + assertAction(t, ActionAbortMultipartUpload, result.Action) + }) + + t.Run("recent_upload_is_not_aborted", func(t *testing.T) { + result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-3*24*time.Hour), now) + assertAction(t, ActionNone, result.Action) + }) + + t.Run("prefix_scoped_abort", func(t *testing.T) { + prefixRules := []Rule{{ + ID: "abort-logs", Status: "Enabled", + Prefix: "logs/", + AbortMPUDaysAfterInitiation: 1, + }} + result := EvaluateMPUAbort(prefixRules, "data/file.bin", now.Add(-5*24*time.Hour), now) + assertAction(t, ActionNone, result.Action) + }) +} + +func TestExpectedExpiryTime(t *testing.T) { + ref := time.Date(2026, 3, 1, 15, 30, 0, 0, time.UTC) + + t.Run("30_days", func(t *testing.T) { + // S3 spec: expires at midnight UTC of day 32 (ref + 31 days, truncated). + expiry := expectedExpiryTime(ref, 30) + expected := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + if !expiry.Equal(expected) { + t.Errorf("expected %v, got %v", expected, expiry) + } + }) + + t.Run("zero_days_returns_ref", func(t *testing.T) { + expiry := expectedExpiryTime(ref, 0) + if !expiry.Equal(ref) { + t.Errorf("expected %v, got %v", ref, expiry) + } + }) +} + +func assertAction(t *testing.T, expected, actual Action) { + t.Helper() + if expected != actual { + t.Errorf("expected action %d, got %d", expected, actual) + } +} + +func assertEqual(t *testing.T, expected, actual string) { + t.Helper() + if expected != actual { + t.Errorf("expected %q, got %q", expected, actual) + } +} diff --git a/weed/s3api/s3lifecycle/filter.go b/weed/s3api/s3lifecycle/filter.go new file mode 100644 index 000000000..d01e6f731 --- /dev/null +++ b/weed/s3api/s3lifecycle/filter.go @@ -0,0 +1,56 @@ +package s3lifecycle + +import "strings" + +// matchesFilter checks if an object matches the rule's filter criteria +// (prefix, tags, and size constraints). +func matchesFilter(rule Rule, obj ObjectInfo) bool { + if !matchesPrefix(rule.Prefix, obj.Key) { + return false + } + if !matchesTags(rule.FilterTags, obj.Tags) { + return false + } + if !matchesSize(rule.FilterSizeGreaterThan, rule.FilterSizeLessThan, obj.Size) { + return false + } + return true +} + +// matchesPrefix returns true if the object key starts with the given prefix. +// An empty prefix matches all keys. +func matchesPrefix(prefix, key string) bool { + if prefix == "" { + return true + } + return strings.HasPrefix(key, prefix) +} + +// matchesTags returns true if all rule tags are present in the object's tags +// with matching values. An empty or nil rule tag set matches all objects. +func matchesTags(ruleTags, objTags map[string]string) bool { + if len(ruleTags) == 0 { + return true + } + if len(objTags) == 0 { + return false + } + for k, v := range ruleTags { + if objVal, ok := objTags[k]; !ok || objVal != v { + return false + } + } + return true +} + +// matchesSize returns true if the object's size falls within the specified +// bounds. Zero values mean no constraint on that side. +func matchesSize(greaterThan, lessThan, objSize int64) bool { + if greaterThan > 0 && objSize <= greaterThan { + return false + } + if lessThan > 0 && objSize >= lessThan { + return false + } + return true +} diff --git a/weed/s3api/s3lifecycle/filter_test.go b/weed/s3api/s3lifecycle/filter_test.go new file mode 100644 index 000000000..c8bcfeb10 --- /dev/null +++ b/weed/s3api/s3lifecycle/filter_test.go @@ -0,0 +1,79 @@ +package s3lifecycle + +import "testing" + +func TestMatchesPrefix(t *testing.T) { + tests := []struct { + name string + prefix string + key string + want bool + }{ + {"empty_prefix_matches_all", "", "any/key.txt", true}, + {"exact_prefix_match", "logs/", "logs/app.log", true}, + {"prefix_mismatch", "logs/", "data/file.txt", false}, + {"key_shorter_than_prefix", "very/long/prefix/", "short", false}, + {"prefix_equals_key", "exact", "exact", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := matchesPrefix(tt.prefix, tt.key); got != tt.want { + t.Errorf("matchesPrefix(%q, %q) = %v, want %v", tt.prefix, tt.key, got, tt.want) + } + }) + } +} + +func TestMatchesTags(t *testing.T) { + tests := []struct { + name string + ruleTags map[string]string + objTags map[string]string + want bool + }{ + {"nil_rule_tags_match_all", nil, map[string]string{"a": "1"}, true}, + {"empty_rule_tags_match_all", map[string]string{}, map[string]string{"a": "1"}, true}, + {"nil_obj_tags_no_match", map[string]string{"a": "1"}, nil, false}, + {"single_tag_match", map[string]string{"env": "dev"}, map[string]string{"env": "dev", "foo": "bar"}, true}, + {"single_tag_value_mismatch", map[string]string{"env": "dev"}, map[string]string{"env": "prod"}, false}, + {"single_tag_key_missing", map[string]string{"env": "dev"}, map[string]string{"foo": "bar"}, false}, + {"multi_tag_all_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev", "tier": "hot", "extra": "x"}, true}, + {"multi_tag_partial_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := matchesTags(tt.ruleTags, tt.objTags); got != tt.want { + t.Errorf("matchesTags() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMatchesSize(t *testing.T) { + tests := []struct { + name string + greaterThan int64 + lessThan int64 + objSize int64 + want bool + }{ + {"no_constraints", 0, 0, 1000, true}, + {"only_greater_than_pass", 100, 0, 200, true}, + {"only_greater_than_fail", 100, 0, 50, false}, + {"only_greater_than_equal_fail", 100, 0, 100, false}, + {"only_less_than_pass", 0, 1000, 500, true}, + {"only_less_than_fail", 0, 1000, 2000, false}, + {"only_less_than_equal_fail", 0, 1000, 1000, false}, + {"both_constraints_pass", 100, 1000, 500, true}, + {"both_constraints_too_small", 100, 1000, 50, false}, + {"both_constraints_too_large", 100, 1000, 2000, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := matchesSize(tt.greaterThan, tt.lessThan, tt.objSize); got != tt.want { + t.Errorf("matchesSize(%d, %d, %d) = %v, want %v", + tt.greaterThan, tt.lessThan, tt.objSize, got, tt.want) + } + }) + } +} diff --git a/weed/s3api/s3lifecycle/rule.go b/weed/s3api/s3lifecycle/rule.go new file mode 100644 index 000000000..4900e5c02 --- /dev/null +++ b/weed/s3api/s3lifecycle/rule.go @@ -0,0 +1,95 @@ +package s3lifecycle + +import "time" + +// Rule is a flattened, evaluator-friendly representation of an S3 lifecycle rule. +// Callers convert from the XML-parsed s3api.Rule (which has nested structs with +// set-flags for conditional XML marshaling) to this type. +type Rule struct { + ID string + Status string // "Enabled" or "Disabled" + + // Prefix filter (from Rule.Prefix or Rule.Filter.Prefix or Rule.Filter.And.Prefix). + Prefix string + + // Expiration for current versions. + ExpirationDays int + ExpirationDate time.Time + ExpiredObjectDeleteMarker bool + + // Expiration for non-current versions. + NoncurrentVersionExpirationDays int + NewerNoncurrentVersions int + + // Abort incomplete multipart uploads. + AbortMPUDaysAfterInitiation int + + // Tag filter (from Rule.Filter.Tag or Rule.Filter.And.Tags). + FilterTags map[string]string + + // Size filters. + FilterSizeGreaterThan int64 + FilterSizeLessThan int64 +} + +// ObjectInfo is the metadata about an object that the evaluator uses to +// determine which lifecycle action applies. Callers build this from filer +// entry attributes and extended metadata. +type ObjectInfo struct { + // Key is the object key relative to the bucket root. + Key string + + // ModTime is the object's modification time (entry.Attributes.Mtime). + ModTime time.Time + + // Size is the object size in bytes (entry.Attributes.FileSize). + Size int64 + + // IsLatest is true if this is the current version of the object. + IsLatest bool + + // IsDeleteMarker is true if this entry is an S3 delete marker. + IsDeleteMarker bool + + // NumVersions is the total number of versions for this object key, + // including delete markers. Used for ExpiredObjectDeleteMarker evaluation. + NumVersions int + + // SuccessorModTime is the creation time of the version that replaced + // this one (making it non-current). Derived from the successor's version + // ID timestamp. Zero value for the latest version. + SuccessorModTime time.Time + + // NoncurrentIndex is the 0-based position among non-current versions + // sorted newest-first (0 = newest non-current version). Used by + // NewerNoncurrentVersions evaluation. -1 or unset for current versions. + NoncurrentIndex int + + // Tags are the object's user-defined tags, extracted from the entry's + // Extended metadata (keys prefixed with "X-Amz-Tagging-"). + Tags map[string]string +} + +// Action represents the lifecycle action to take on an object. +type Action int + +const ( + // ActionNone means no lifecycle rule applies. + ActionNone Action = iota + // ActionDeleteObject deletes the current version of the object. + ActionDeleteObject + // ActionDeleteVersion deletes a specific non-current version. + ActionDeleteVersion + // ActionExpireDeleteMarker removes a delete marker that is the sole remaining version. + ActionExpireDeleteMarker + // ActionAbortMultipartUpload aborts an incomplete multipart upload. + ActionAbortMultipartUpload +) + +// EvalResult is the output of lifecycle rule evaluation. +type EvalResult struct { + // Action is the lifecycle action to take. + Action Action + // RuleID is the ID of the rule that triggered this action. + RuleID string +} diff --git a/weed/s3api/s3lifecycle/tags.go b/weed/s3api/s3lifecycle/tags.go new file mode 100644 index 000000000..57092ed56 --- /dev/null +++ b/weed/s3api/s3lifecycle/tags.go @@ -0,0 +1,34 @@ +package s3lifecycle + +import "strings" + +const tagPrefix = "X-Amz-Tagging-" + +// ExtractTags extracts S3 object tags from a filer entry's Extended metadata. +// Tags are stored with the key prefix "X-Amz-Tagging-" followed by the tag key. +func ExtractTags(extended map[string][]byte) map[string]string { + if len(extended) == 0 { + return nil + } + var tags map[string]string + for k, v := range extended { + if strings.HasPrefix(k, tagPrefix) { + if tags == nil { + tags = make(map[string]string) + } + tags[k[len(tagPrefix):]] = string(v) + } + } + return tags +} + +// HasTagRules returns true if any enabled rule in the set uses tag-based filtering. +// This is used as an optimization to skip tag extraction when no rules need it. +func HasTagRules(rules []Rule) bool { + for _, r := range rules { + if r.Status == "Enabled" && len(r.FilterTags) > 0 { + return true + } + } + return false +} diff --git a/weed/s3api/s3lifecycle/tags_test.go b/weed/s3api/s3lifecycle/tags_test.go new file mode 100644 index 000000000..0eb198c5f --- /dev/null +++ b/weed/s3api/s3lifecycle/tags_test.go @@ -0,0 +1,89 @@ +package s3lifecycle + +import "testing" + +func TestExtractTags(t *testing.T) { + t.Run("extracts_tags_with_prefix", func(t *testing.T) { + extended := map[string][]byte{ + "X-Amz-Tagging-env": []byte("prod"), + "X-Amz-Tagging-project": []byte("foo"), + "Content-Type": []byte("text/plain"), + "X-Amz-Meta-Custom": []byte("value"), + } + tags := ExtractTags(extended) + if len(tags) != 2 { + t.Fatalf("expected 2 tags, got %d", len(tags)) + } + if tags["env"] != "prod" { + t.Errorf("expected env=prod, got %q", tags["env"]) + } + if tags["project"] != "foo" { + t.Errorf("expected project=foo, got %q", tags["project"]) + } + }) + + t.Run("nil_extended_returns_nil", func(t *testing.T) { + tags := ExtractTags(nil) + if tags != nil { + t.Errorf("expected nil, got %v", tags) + } + }) + + t.Run("no_tags_returns_nil", func(t *testing.T) { + extended := map[string][]byte{ + "Content-Type": []byte("text/plain"), + } + tags := ExtractTags(extended) + if tags != nil { + t.Errorf("expected nil, got %v", tags) + } + }) + + t.Run("empty_tag_value", func(t *testing.T) { + extended := map[string][]byte{ + "X-Amz-Tagging-empty": []byte(""), + } + tags := ExtractTags(extended) + if len(tags) != 1 { + t.Fatalf("expected 1 tag, got %d", len(tags)) + } + if tags["empty"] != "" { + t.Errorf("expected empty value, got %q", tags["empty"]) + } + }) +} + +func TestHasTagRules(t *testing.T) { + t.Run("has_tag_rules", func(t *testing.T) { + rules := []Rule{ + {Status: "Enabled", FilterTags: map[string]string{"env": "dev"}}, + } + if !HasTagRules(rules) { + t.Error("expected true") + } + }) + + t.Run("no_tag_rules", func(t *testing.T) { + rules := []Rule{ + {Status: "Enabled", ExpirationDays: 30}, + } + if HasTagRules(rules) { + t.Error("expected false") + } + }) + + t.Run("disabled_tag_rule", func(t *testing.T) { + rules := []Rule{ + {Status: "Disabled", FilterTags: map[string]string{"env": "dev"}}, + } + if HasTagRules(rules) { + t.Error("expected false for disabled rule") + } + }) + + t.Run("empty_rules", func(t *testing.T) { + if HasTagRules(nil) { + t.Error("expected false for nil rules") + } + }) +} diff --git a/weed/s3api/s3lifecycle/version_time.go b/weed/s3api/s3lifecycle/version_time.go new file mode 100644 index 000000000..d4f4c5f94 --- /dev/null +++ b/weed/s3api/s3lifecycle/version_time.go @@ -0,0 +1,42 @@ +package s3lifecycle + +import ( + "math" + "strconv" + "time" +) + +// versionIdFormatThreshold distinguishes old vs new format version IDs. +// New format (inverted timestamps) produces values above this threshold; +// old format (raw timestamps) produces values below it. +const versionIdFormatThreshold = 0x4000000000000000 + +// GetVersionTimestamp extracts the actual timestamp from a SeaweedFS version ID, +// handling both old (raw nanosecond) and new (inverted nanosecond) formats. +// Returns zero time if the version ID is invalid or "null". +func GetVersionTimestamp(versionId string) time.Time { + ns := getVersionTimestampNanos(versionId) + if ns == 0 { + return time.Time{} + } + return time.Unix(0, ns) +} + +// getVersionTimestampNanos extracts the raw nanosecond timestamp from a version ID. +func getVersionTimestampNanos(versionId string) int64 { + if len(versionId) < 16 || versionId == "null" { + return 0 + } + timestampPart, err := strconv.ParseUint(versionId[:16], 16, 64) + if err != nil { + return 0 + } + if timestampPart > math.MaxInt64 { + return 0 + } + if timestampPart > versionIdFormatThreshold { + // New format: inverted timestamp, convert back. + return int64(math.MaxInt64 - timestampPart) + } + return int64(timestampPart) +} diff --git a/weed/s3api/s3lifecycle/version_time_test.go b/weed/s3api/s3lifecycle/version_time_test.go new file mode 100644 index 000000000..460cbec58 --- /dev/null +++ b/weed/s3api/s3lifecycle/version_time_test.go @@ -0,0 +1,74 @@ +package s3lifecycle + +import ( + "fmt" + "math" + "testing" + "time" +) + +func TestGetVersionTimestamp(t *testing.T) { + t.Run("new_format_inverted_timestamp", func(t *testing.T) { + // Simulate a new-format version ID (inverted timestamp above threshold). + now := time.Now() + inverted := math.MaxInt64 - now.UnixNano() + versionId := fmt.Sprintf("%016x", inverted) + "0000000000000000" + + got := GetVersionTimestamp(versionId) + // Should recover the original timestamp within 1 second. + diff := got.Sub(now) + if diff < -time.Second || diff > time.Second { + t.Errorf("timestamp diff too large: %v (got %v, want ~%v)", diff, got, now) + } + }) + + t.Run("old_format_raw_timestamp", func(t *testing.T) { + // Simulate an old-format version ID (raw nanosecond timestamp below threshold). + // Use a timestamp from 2023 which would be below threshold. + ts := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC) + versionId := fmt.Sprintf("%016x", ts.UnixNano()) + "abcdef0123456789" + + got := GetVersionTimestamp(versionId) + if !got.Equal(ts) { + t.Errorf("expected %v, got %v", ts, got) + } + }) + + t.Run("null_version_id", func(t *testing.T) { + got := GetVersionTimestamp("null") + if !got.IsZero() { + t.Errorf("expected zero time for null version, got %v", got) + } + }) + + t.Run("empty_version_id", func(t *testing.T) { + got := GetVersionTimestamp("") + if !got.IsZero() { + t.Errorf("expected zero time for empty version, got %v", got) + } + }) + + t.Run("short_version_id", func(t *testing.T) { + got := GetVersionTimestamp("abc") + if !got.IsZero() { + t.Errorf("expected zero time for short version, got %v", got) + } + }) + + t.Run("high_bit_overflow_returns_zero", func(t *testing.T) { + // Version ID with first 16 hex chars > math.MaxInt64 should return zero, + // not a wrapped negative timestamp. + versionId := "80000000000000000000000000000000" + got := GetVersionTimestamp(versionId) + if !got.IsZero() { + t.Errorf("expected zero time for overflow version ID, got %v", got) + } + }) + + t.Run("invalid_hex", func(t *testing.T) { + got := GetVersionTimestamp("zzzzzzzzzzzzzzzz0000000000000000") + if !got.IsZero() { + t.Errorf("expected zero time for invalid hex, got %v", got) + } + }) +}