From 9b6fc499465399fbc3ccaf8307d484c380f51331 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 26 Feb 2026 11:56:10 -0800 Subject: [PATCH] Chart createBuckets config #8368: Add TTL, Object Lock, and Versioning support (#8375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Chart createBuckets config #8368: Add TTL, Object Lock, and Versioning support * Update weed/shell/command_s3_bucket_versioning.go Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * address comments * address comments * go fmt * fix failures are still treated like “bucket not found” --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../shared/post-install-bucket-hook.yaml | 54 +++++- k8s/charts/seaweedfs/values.yaml | 18 ++ weed/shell/command_s3_bucket_versioning.go | 167 ++++++++++++++++++ 3 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 weed/shell/command_s3_bucket_versioning.go diff --git a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml index 930c474e9..f365548c0 100644 --- a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml +++ b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml @@ -3,6 +3,15 @@ {{- $s3Enabled := false }} {{- $enableAuth := false }} {{- $existingConfigSecret := "" }} +{{- $bucketsFolder := "/buckets" }} +{{- $bucketEnvVars := merge (dict) (.Values.global.extraEnvironmentVars | default dict) }} +{{- if .Values.allInOne.enabled }} + {{- $bucketEnvVars = merge (.Values.allInOne.extraEnvironmentVars | default dict) $bucketEnvVars }} +{{- else }} + {{- $bucketEnvVars = merge (.Values.filer.extraEnvironmentVars | default dict) $bucketEnvVars }} +{{- end }} +{{- $bucketsFolder = default $bucketsFolder (get $bucketEnvVars "WEED_FILER_BUCKETS_FOLDER") }} +{{- $bucketsFolder = trimSuffix "/" $bucketsFolder }} {{- /* Check allInOne mode first */}} {{- if .Values.allInOne.enabled }} @@ -108,15 +117,52 @@ spec: wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}" wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}" {{- end }} + set -o pipefail {{- range $createBuckets }} - if /bin/echo "s3.bucket.list" | /usr/bin/weed shell | awk '{print $1}' | grep -Fxq "{{ .name }}"; then - echo "Bucket '{{ .name }}' already exists, skipping creation." + {{- $bucketName := .name }} + {{- $bucketLock := or .lock .objectLock .withLock }} + bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; } + if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then + echo "Bucket '{{ $bucketName }}' already exists, skipping creation." else - echo "Creating bucket '{{ .name }}'..." - /bin/echo "s3.bucket.create --name {{ .name }}" | /usr/bin/weed shell + echo "Creating bucket '{{ $bucketName }}'..." + /bin/echo 's3.bucket.create --name {{ $bucketName }}{{- if $bucketLock }} --withLock{{- end }}' | /usr/bin/weed shell fi {{- end }} {{- range $createBuckets }} + {{- $bucketLock := or .lock .objectLock .withLock }} + {{- if $bucketLock }} + /bin/echo 's3.bucket.lock -name {{ .name }} -enable' | /usr/bin/weed shell + {{- end }} + {{- end }} + {{- range $createBuckets }} + {{- $bucketVersioning := "" }} + {{- if kindIs "bool" .versioning }} + {{- if .versioning }} + {{- $bucketVersioning = "Enabled" }} + {{- end }} + {{- else if kindIs "string" .versioning }} + {{- $versioningLower := lower .versioning }} + {{- if eq $versioningLower "enabled" "enable" "true" }} + {{- $bucketVersioning = "Enabled" }} + {{- else if eq $versioningLower "suspended" "disable" "false" }} + {{- $bucketVersioning = "Suspended" }} + {{- else if or (eq .versioning "Enabled") (eq .versioning "Suspended") }} + {{- $bucketVersioning = .versioning }} + {{- else }} + {{- fail (printf "Invalid versioning value for bucket %s: %s. Must be 'Enabled' or 'Suspended'" .name .versioning) }} + {{- end }} + {{- end }} + {{- if $bucketVersioning }} + /bin/echo 's3.bucket.versioning -name {{ .name }} -status {{ $bucketVersioning }}' | /usr/bin/weed shell + {{- end }} + {{- end }} + {{- range $createBuckets }} + {{- if .ttl }} + /bin/echo 'fs.configure -locationPrefix={{ $bucketsFolder }}/{{ .name }}/ -ttl={{ .ttl }} -apply' | /usr/bin/weed shell + {{- end }} + {{- end }} + {{- range $createBuckets }} {{- if .anonymousRead }} /bin/echo \ "s3.configure --user anonymous \ diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 3162ddd33..84c637c75 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -898,9 +898,15 @@ filer: auditLogConfig: {} # You may specify buckets to be created during the install or upgrade process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` + # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d) + # objectLock enables S3 Object Lock (irreversible, forces versioning) + # versioning: Enabled or Suspended (or true to enable) # createBuckets: # - name: bucket-a # anonymousRead: true + # ttl: 7d + # objectLock: true + # versioning: Enabled # - name: bucket-b # anonymousRead: false @@ -936,9 +942,15 @@ s3: auditLogConfig: {} # You may specify buckets to be created during the install or upgrade process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` + # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d) + # objectLock enables S3 Object Lock (irreversible, forces versioning) + # versioning: Enabled or Suspended (or true to enable) # createBuckets: # - name: bucket-a # anonymousRead: true + # ttl: 7d + # objectLock: true + # versioning: Enabled # - name: bucket-b # anonymousRead: false @@ -1443,9 +1455,15 @@ allInOne: auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig) # You may specify buckets to be created during the install process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` + # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d) + # objectLock enables S3 Object Lock (irreversible, forces versioning) + # versioning: Enabled or Suspended (or true to enable) # createBuckets: # - name: bucket-a # anonymousRead: true + # ttl: 7d + # objectLock: true + # versioning: Enabled # - name: bucket-b # anonymousRead: false diff --git a/weed/shell/command_s3_bucket_versioning.go b/weed/shell/command_s3_bucket_versioning.go new file mode 100644 index 000000000..b324a33dd --- /dev/null +++ b/weed/shell/command_s3_bucket_versioning.go @@ -0,0 +1,167 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "io" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +func init() { + Commands = append(Commands, &commandS3BucketVersioning{}) +} + +type commandS3BucketVersioning struct { +} + +func (c *commandS3BucketVersioning) Name() string { + return "s3.bucket.versioning" +} + +func (c *commandS3BucketVersioning) Help() string { + return `view or update S3 bucket versioning configuration + + Example: + # View the current versioning status + s3.bucket.versioning -name + + # Enable versioning + s3.bucket.versioning -name -enable + + # Suspend versioning + s3.bucket.versioning -name -suspend + + # Set versioning status explicitly (Enabled or Suspended) + s3.bucket.versioning -name -status Enabled + + Object Lock requires versioning to be enabled and prevents suspending it. +` +} + +func (c *commandS3BucketVersioning) HasTag(CommandTag) bool { + return false +} + +func (c *commandS3BucketVersioning) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + bucketName := bucketCommand.String("name", "", "bucket name") + statusFlag := bucketCommand.String("status", "", "versioning status: Enabled or Suspended") + enable := bucketCommand.Bool("enable", false, "enable versioning on the bucket") + suspend := bucketCommand.Bool("suspend", false, "suspend versioning on the bucket") + if err = bucketCommand.Parse(args); err != nil { + return err + } + + if *bucketName == "" { + return fmt.Errorf("empty bucket name") + } + + desiredStatus, err := normalizeVersioningStatus(*statusFlag, *enable, *suspend) + if err != nil { + return err + } + + err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer configuration: %w", err) + } + filerBucketsPath := resp.DirBuckets + + lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ + Directory: filerBucketsPath, + Name: *bucketName, + }) + if err != nil { + return fmt.Errorf("lookup bucket %s: %w", *bucketName, err) + } + + entry := lookupResp.Entry + currentStatus, lockEnabled := getBucketVersioningState(entry) + + if desiredStatus == "" { + if lockEnabled { + currentStatus = s3_constants.VersioningEnabled + } + fmt.Fprintf(writer, "Bucket: %s\n", *bucketName) + switch currentStatus { + case s3_constants.VersioningEnabled: + fmt.Fprintf(writer, "Versioning: Enabled\n") + case s3_constants.VersioningSuspended: + fmt.Fprintf(writer, "Versioning: Suspended\n") + case "": + fmt.Fprintf(writer, "Versioning: Unconfigured\n") + default: + fmt.Fprintf(writer, "Versioning: %s\n", currentStatus) + } + if lockEnabled { + fmt.Fprintf(writer, "Object Lock: Enabled\n") + } + return nil + } + + if lockEnabled && desiredStatus == s3_constants.VersioningSuspended { + return fmt.Errorf("cannot suspend versioning on bucket %s: Object Lock is enabled", *bucketName) + } + + if entry.Extended == nil { + entry.Extended = make(map[string][]byte) + } + entry.Extended[s3_constants.ExtVersioningKey] = []byte(desiredStatus) + + if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{ + Directory: filerBucketsPath, + Entry: entry, + }); err != nil { + return fmt.Errorf("failed to update bucket: %w", err) + } + + fmt.Fprintf(writer, "Bucket %s versioning set to %s\n", *bucketName, desiredStatus) + return nil + }) + + return err +} + +func normalizeVersioningStatus(statusFlag string, enable bool, suspend bool) (string, error) { + if enable && suspend { + return "", fmt.Errorf("only one of -enable or -suspend can be set") + } + if (enable || suspend) && statusFlag != "" { + return "", fmt.Errorf("use either -status or -enable/-suspend, not both") + } + if enable { + return s3_constants.VersioningEnabled, nil + } + if suspend { + return s3_constants.VersioningSuspended, nil + } + if statusFlag == "" { + return "", nil + } + switch strings.ToLower(strings.TrimSpace(statusFlag)) { + case "enabled": + return s3_constants.VersioningEnabled, nil + case "suspended": + return s3_constants.VersioningSuspended, nil + default: + return "", fmt.Errorf("invalid versioning status %q: must be Enabled or Suspended", statusFlag) + } +} + +func getBucketVersioningState(entry *filer_pb.Entry) (status string, lockEnabled bool) { + if entry.Extended == nil { + return "", false + } + if versioning, ok := entry.Extended[s3_constants.ExtVersioningKey]; ok { + status = string(versioning) + } + if lockStatus, ok := entry.Extended[s3_constants.ExtObjectLockEnabledKey]; ok { + lockEnabled = string(lockStatus) == s3_constants.ObjectLockEnabled + } + return status, lockEnabled +}