From 34922fd761bfebabf47f48a54580d99dd523aa85 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 11 Feb 2026 09:45:50 -0800 Subject: [PATCH] s3api: persist lifecycle TTL rules and enforce on assign --- weed/s3api/s3_constants/extend_key.go | 5 +- weed/s3api/s3api_bucket_handlers.go | 18 ++- weed/s3api/s3api_bucket_lifecycle_ttl.go | 124 ++++++++++++++++++ weed/s3api/s3api_bucket_lifecycle_ttl_test.go | 68 ++++++++++ weed/s3api/s3api_object_handlers_put.go | 5 + 5 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 weed/s3api/s3api_bucket_lifecycle_ttl.go create mode 100644 weed/s3api/s3api_bucket_lifecycle_ttl_test.go diff --git a/weed/s3api/s3_constants/extend_key.go b/weed/s3api/s3_constants/extend_key.go index bb3d637d1..3e136dc3a 100644 --- a/weed/s3api/s3_constants/extend_key.go +++ b/weed/s3api/s3_constants/extend_key.go @@ -11,7 +11,7 @@ const ( ExtETagKey = "Seaweed-X-Amz-ETag" ExtLatestVersionIdKey = "Seaweed-X-Amz-Latest-Version-Id" ExtLatestVersionFileNameKey = "Seaweed-X-Amz-Latest-Version-File-Name" - ExtAllowEmptyFolders = "Seaweed-X-Amz-Allow-Empty-Folders" + ExtAllowEmptyFolders = "Seaweed-X-Amz-Allow-Empty-Folders" // Cached list metadata in .versions directory for single-scan efficiency ExtLatestVersionSizeKey = "Seaweed-X-Amz-Latest-Version-Size" ExtLatestVersionETagKey = "Seaweed-X-Amz-Latest-Version-ETag" @@ -21,7 +21,8 @@ const ( ExtMultipartObjectKey = "key" // Bucket Policy - ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy" + ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy" + ExtBucketLifecycleTTLRulesKey = "Seaweed-X-Amz-Bucket-Lifecycle-TTL-Rules" // Object Retention and Legal Hold ExtObjectLockModeKey = "Seaweed-X-Amz-Object-Lock-Mode" diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 2c4a1a884..e4988947a 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -860,6 +860,7 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr collectionName := s3a.getCollectionName(bucket) collectionTtls := fc.GetCollectionTtls(collectionName) changed := false + lifecycleRules := make([]bucketLifecycleTTLRule, 0, len(lifeCycleConfig.Rules)) for _, rule := range lifeCycleConfig.Rules { if rule.Status != Enabled { @@ -879,6 +880,12 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr if rule.Expiration.Days == 0 { continue } + ttlSec := int32((time.Duration(rule.Expiration.Days) * util.LifeCycleInterval).Seconds()) + lifecycleRules = append(lifecycleRules, bucketLifecycleTTLRule{ + Prefix: rulePrefix, + TtlSec: ttlSec, + }) + locationPrefix := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix) locConf := &filer_pb.FilerConf_PathConf{ LocationPrefix: locationPrefix, @@ -893,7 +900,6 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - ttlSec := int32((time.Duration(rule.Expiration.Days) * util.LifeCycleInterval).Seconds()) glog.V(2).Infof("Start updating TTL for %s", locationPrefix) if updErr := s3a.updateEntriesTTL(locationPrefix, ttlSec); updErr != nil { glog.Errorf("PutBucketLifecycleConfigurationHandler update TTL for %s: %s", locationPrefix, updErr) @@ -917,6 +923,11 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr return } } + if err := s3a.persistBucketLifecycleTTLRules(bucket, lifecycleRules); err != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler persist lifecycle metadata for %s: %v", bucket, err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } writeSuccessResponseEmpty(w, r) } @@ -967,6 +978,11 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h return } } + if err := s3a.persistBucketLifecycleTTLRules(bucket, nil); err != nil { + glog.Errorf("DeleteBucketLifecycleHandler clear lifecycle metadata for %s: %v", bucket, err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } s3err.WriteEmptyResponse(w, r, http.StatusNoContent) } diff --git a/weed/s3api/s3api_bucket_lifecycle_ttl.go b/weed/s3api/s3api_bucket_lifecycle_ttl.go new file mode 100644 index 000000000..80eff08b8 --- /dev/null +++ b/weed/s3api/s3api_bucket_lifecycle_ttl.go @@ -0,0 +1,124 @@ +package s3api + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +type bucketLifecycleTTLRule struct { + Prefix string `json:"prefix"` + TtlSec int32 `json:"ttlSec"` +} + +func normalizeLifecycleRulePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + prefix = strings.TrimPrefix(prefix, "/") + if prefix == "." || prefix == "/" { + return "" + } + return prefix +} + +func encodeBucketLifecycleTTLRules(rules []bucketLifecycleTTLRule) ([]byte, error) { + if len(rules) == 0 { + return nil, nil + } + sorted := make([]bucketLifecycleTTLRule, 0, len(rules)) + for _, rule := range rules { + if rule.TtlSec <= 0 { + continue + } + sorted = append(sorted, bucketLifecycleTTLRule{ + Prefix: normalizeLifecycleRulePrefix(rule.Prefix), + TtlSec: rule.TtlSec, + }) + } + if len(sorted) == 0 { + return nil, nil + } + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Prefix < sorted[j].Prefix + }) + return json.Marshal(sorted) +} + +func decodeBucketLifecycleTTLRules(extended map[string][]byte) []bucketLifecycleTTLRule { + if len(extended) == 0 { + return nil + } + serialized := extended[s3_constants.ExtBucketLifecycleTTLRulesKey] + if len(serialized) == 0 { + return nil + } + var rules []bucketLifecycleTTLRule + if err := json.Unmarshal(serialized, &rules); err != nil { + glog.Warningf("decode bucket lifecycle ttl rules: %v", err) + return nil + } + return rules +} + +func matchBucketLifecycleTTLSeconds(rules []bucketLifecycleTTLRule, objectKey string) int32 { + if len(rules) == 0 { + return 0 + } + objectKey = normalizeLifecycleRulePrefix(objectKey) + var bestPrefix string + var ttlSeconds int32 + for _, rule := range rules { + prefix := normalizeLifecycleRulePrefix(rule.Prefix) + if prefix != "" && !strings.HasPrefix(objectKey, prefix) { + continue + } + if len(prefix) >= len(bestPrefix) { + bestPrefix = prefix + ttlSeconds = rule.TtlSec + } + } + return ttlSeconds +} + +func (s3a *S3ApiServer) resolveBucketLifecycleTTLSeconds(bucket, filePath string) int32 { + config, errCode := s3a.getBucketConfig(bucket) + if errCode != s3err.ErrNone || config == nil || config.Entry == nil { + return 0 + } + bucketPrefix := s3a.bucketDir(bucket) + "/" + objectKey := strings.TrimPrefix(filePath, bucketPrefix) + if objectKey == filePath { + return 0 + } + rules := decodeBucketLifecycleTTLRules(config.Entry.Extended) + return matchBucketLifecycleTTLSeconds(rules, objectKey) +} + +func (s3a *S3ApiServer) persistBucketLifecycleTTLRules(bucket string, rules []bucketLifecycleTTLRule) error { + serialized, err := encodeBucketLifecycleTTLRules(rules) + if err != nil { + return fmt.Errorf("encode lifecycle ttl rules: %w", err) + } + errCode := s3a.updateBucketConfig(bucket, func(config *BucketConfig) error { + if config.Entry == nil { + return fmt.Errorf("bucket %s has no entry", bucket) + } + if config.Entry.Extended == nil { + config.Entry.Extended = make(map[string][]byte) + } + if len(serialized) == 0 { + delete(config.Entry.Extended, s3_constants.ExtBucketLifecycleTTLRulesKey) + } else { + config.Entry.Extended[s3_constants.ExtBucketLifecycleTTLRulesKey] = serialized + } + return nil + }) + if errCode != s3err.ErrNone { + return fmt.Errorf("persist lifecycle ttl rules: %v", errCode) + } + return nil +} diff --git a/weed/s3api/s3api_bucket_lifecycle_ttl_test.go b/weed/s3api/s3api_bucket_lifecycle_ttl_test.go new file mode 100644 index 000000000..5ade6308c --- /dev/null +++ b/weed/s3api/s3api_bucket_lifecycle_ttl_test.go @@ -0,0 +1,68 @@ +package s3api + +import ( + "encoding/json" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +func TestMatchBucketLifecycleTTLSeconds(t *testing.T) { + rules := []bucketLifecycleTTLRule{ + {Prefix: "", TtlSec: 3600}, + {Prefix: "logs/", TtlSec: 7200}, + {Prefix: "logs/archive/", TtlSec: 10800}, + } + + if got := matchBucketLifecycleTTLSeconds(rules, "logs/file.txt"); got != 7200 { + t.Fatalf("expected 7200 for logs/file.txt, got %d", got) + } + if got := matchBucketLifecycleTTLSeconds(rules, "logs/archive/file.txt"); got != 10800 { + t.Fatalf("expected 10800 for logs/archive/file.txt, got %d", got) + } + if got := matchBucketLifecycleTTLSeconds(rules, "other/file.txt"); got != 3600 { + t.Fatalf("expected 3600 for other/file.txt, got %d", got) + } +} + +func TestDecodeBucketLifecycleTTLRules(t *testing.T) { + extended := map[string][]byte{ + s3_constants.ExtBucketLifecycleTTLRulesKey: []byte(`[{"prefix":"logs/","ttlSec":86400}]`), + } + rules := decodeBucketLifecycleTTLRules(extended) + if len(rules) != 1 { + t.Fatalf("expected 1 rule, got %d", len(rules)) + } + if rules[0].Prefix != "logs/" || rules[0].TtlSec != 86400 { + t.Fatalf("unexpected rule: %+v", rules[0]) + } + + extended[s3_constants.ExtBucketLifecycleTTLRulesKey] = []byte(`{invalid json`) + if got := decodeBucketLifecycleTTLRules(extended); got != nil { + t.Fatalf("expected nil rules for invalid JSON, got %+v", got) + } +} + +func TestEncodeBucketLifecycleTTLRules(t *testing.T) { + serialized, err := encodeBucketLifecycleTTLRules([]bucketLifecycleTTLRule{ + {Prefix: "/logs/", TtlSec: 86400}, + {Prefix: "tmp/", TtlSec: 0}, + {Prefix: "", TtlSec: 3600}, + }) + if err != nil { + t.Fatalf("encode failed: %v", err) + } + var decoded []bucketLifecycleTTLRule + if err := json.Unmarshal(serialized, &decoded); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if len(decoded) != 2 { + t.Fatalf("expected 2 persisted rules, got %d", len(decoded)) + } + if decoded[0].Prefix != "" || decoded[0].TtlSec != 3600 { + t.Fatalf("unexpected first rule: %+v", decoded[0]) + } + if decoded[1].Prefix != "logs/" || decoded[1].TtlSec != 86400 { + t.Fatalf("unexpected second rule: %+v", decoded[1]) + } +} diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 7903bd6a1..89ad49a0f 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -358,6 +358,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader if s3a.option.FilerGroup != "" { collection = s3a.getCollectionName(bucket) } + ttlSeconds := s3a.resolveBucketLifecycleTTLSeconds(bucket, filePath) // Create assign function for chunked upload assignFunc := func(ctx context.Context, count int) (*operation.VolumeAssignRequest, *operation.AssignResult, error) { @@ -370,6 +371,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader DiskType: "", DataCenter: s3a.option.DataCenter, Path: filePath, + TtlSec: ttlSeconds, }) if err != nil { return fmt.Errorf("assign volume: %w", err) @@ -534,6 +536,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader Chunks: chunkResult.FileChunks, // All chunks from auto-chunking Extended: make(map[string][]byte), } + if ttlSeconds > 0 { + entry.Attributes.TtlSec = ttlSeconds + } // Always set Md5 attribute for regular object uploads (PutObject) // This ensures the ETag is a pure MD5 hash, which AWS S3 SDKs expect