Browse Source

Chart createBuckets config #8368: Add TTL, Object Lock, and Versioning support (#8375)

* Chart createBuckets config #8368: Add TTL, Object Lock, and Versioning support

* Update weed/shell/command_s3_bucket_versioning.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* address comments

* address comments

* go fmt

* fix failures are still treated like “bucket not found”

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
pull/8372/merge
Chris Lu 2 days ago
committed by GitHub
parent
commit
9b6fc49946
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 54
      k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml
  2. 18
      k8s/charts/seaweedfs/values.yaml
  3. 167
      weed/shell/command_s3_bucket_versioning.go

54
k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@ -3,6 +3,15 @@
{{- $s3Enabled := false }}
{{- $enableAuth := false }}
{{- $existingConfigSecret := "" }}
{{- $bucketsFolder := "/buckets" }}
{{- $bucketEnvVars := merge (dict) (.Values.global.extraEnvironmentVars | default dict) }}
{{- if .Values.allInOne.enabled }}
{{- $bucketEnvVars = merge (.Values.allInOne.extraEnvironmentVars | default dict) $bucketEnvVars }}
{{- else }}
{{- $bucketEnvVars = merge (.Values.filer.extraEnvironmentVars | default dict) $bucketEnvVars }}
{{- end }}
{{- $bucketsFolder = default $bucketsFolder (get $bucketEnvVars "WEED_FILER_BUCKETS_FOLDER") }}
{{- $bucketsFolder = trimSuffix "/" $bucketsFolder }}
{{- /* Check allInOne mode first */}}
{{- if .Values.allInOne.enabled }}
@ -108,15 +117,52 @@ spec:
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
{{- end }}
set -o pipefail
{{- range $createBuckets }}
if /bin/echo "s3.bucket.list" | /usr/bin/weed shell | awk '{print $1}' | grep -Fxq "{{ .name }}"; then
echo "Bucket '{{ .name }}' already exists, skipping creation."
{{- $bucketName := .name }}
{{- $bucketLock := or .lock .objectLock .withLock }}
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
else
echo "Creating bucket '{{ .name }}'..."
/bin/echo "s3.bucket.create --name {{ .name }}" | /usr/bin/weed shell
echo "Creating bucket '{{ $bucketName }}'..."
/bin/echo 's3.bucket.create --name {{ $bucketName }}{{- if $bucketLock }} --withLock{{- end }}' | /usr/bin/weed shell
fi
{{- end }}
{{- range $createBuckets }}
{{- $bucketLock := or .lock .objectLock .withLock }}
{{- if $bucketLock }}
/bin/echo 's3.bucket.lock -name {{ .name }} -enable' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- $bucketVersioning := "" }}
{{- if kindIs "bool" .versioning }}
{{- if .versioning }}
{{- $bucketVersioning = "Enabled" }}
{{- end }}
{{- else if kindIs "string" .versioning }}
{{- $versioningLower := lower .versioning }}
{{- if eq $versioningLower "enabled" "enable" "true" }}
{{- $bucketVersioning = "Enabled" }}
{{- else if eq $versioningLower "suspended" "disable" "false" }}
{{- $bucketVersioning = "Suspended" }}
{{- else if or (eq .versioning "Enabled") (eq .versioning "Suspended") }}
{{- $bucketVersioning = .versioning }}
{{- else }}
{{- fail (printf "Invalid versioning value for bucket %s: %s. Must be 'Enabled' or 'Suspended'" .name .versioning) }}
{{- end }}
{{- end }}
{{- if $bucketVersioning }}
/bin/echo 's3.bucket.versioning -name {{ .name }} -status {{ $bucketVersioning }}' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- if .ttl }}
/bin/echo 'fs.configure -locationPrefix={{ $bucketsFolder }}/{{ .name }}/ -ttl={{ .ttl }} -apply' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- if .anonymousRead }}
/bin/echo \
"s3.configure --user anonymous \

18
k8s/charts/seaweedfs/values.yaml

@ -898,9 +898,15 @@ filer:
auditLogConfig: {}
# You may specify buckets to be created during the install or upgrade process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false
@ -936,9 +942,15 @@ s3:
auditLogConfig: {}
# You may specify buckets to be created during the install or upgrade process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false
@ -1443,9 +1455,15 @@ allInOne:
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
# You may specify buckets to be created during the install process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false

167
weed/shell/command_s3_bucket_versioning.go

@ -0,0 +1,167 @@
package shell
import (
"context"
"flag"
"fmt"
"io"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
func init() {
Commands = append(Commands, &commandS3BucketVersioning{})
}
type commandS3BucketVersioning struct {
}
func (c *commandS3BucketVersioning) Name() string {
return "s3.bucket.versioning"
}
func (c *commandS3BucketVersioning) Help() string {
return `view or update S3 bucket versioning configuration
Example:
# View the current versioning status
s3.bucket.versioning -name <bucket_name>
# Enable versioning
s3.bucket.versioning -name <bucket_name> -enable
# Suspend versioning
s3.bucket.versioning -name <bucket_name> -suspend
# Set versioning status explicitly (Enabled or Suspended)
s3.bucket.versioning -name <bucket_name> -status Enabled
Object Lock requires versioning to be enabled and prevents suspending it.
`
}
func (c *commandS3BucketVersioning) HasTag(CommandTag) bool {
return false
}
func (c *commandS3BucketVersioning) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
bucketName := bucketCommand.String("name", "", "bucket name")
statusFlag := bucketCommand.String("status", "", "versioning status: Enabled or Suspended")
enable := bucketCommand.Bool("enable", false, "enable versioning on the bucket")
suspend := bucketCommand.Bool("suspend", false, "suspend versioning on the bucket")
if err = bucketCommand.Parse(args); err != nil {
return err
}
if *bucketName == "" {
return fmt.Errorf("empty bucket name")
}
desiredStatus, err := normalizeVersioningStatus(*statusFlag, *enable, *suspend)
if err != nil {
return err
}
err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer configuration: %w", err)
}
filerBucketsPath := resp.DirBuckets
lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
Directory: filerBucketsPath,
Name: *bucketName,
})
if err != nil {
return fmt.Errorf("lookup bucket %s: %w", *bucketName, err)
}
entry := lookupResp.Entry
currentStatus, lockEnabled := getBucketVersioningState(entry)
if desiredStatus == "" {
if lockEnabled {
currentStatus = s3_constants.VersioningEnabled
}
fmt.Fprintf(writer, "Bucket: %s\n", *bucketName)
switch currentStatus {
case s3_constants.VersioningEnabled:
fmt.Fprintf(writer, "Versioning: Enabled\n")
case s3_constants.VersioningSuspended:
fmt.Fprintf(writer, "Versioning: Suspended\n")
case "":
fmt.Fprintf(writer, "Versioning: Unconfigured\n")
default:
fmt.Fprintf(writer, "Versioning: %s\n", currentStatus)
}
if lockEnabled {
fmt.Fprintf(writer, "Object Lock: Enabled\n")
}
return nil
}
if lockEnabled && desiredStatus == s3_constants.VersioningSuspended {
return fmt.Errorf("cannot suspend versioning on bucket %s: Object Lock is enabled", *bucketName)
}
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtVersioningKey] = []byte(desiredStatus)
if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
Directory: filerBucketsPath,
Entry: entry,
}); err != nil {
return fmt.Errorf("failed to update bucket: %w", err)
}
fmt.Fprintf(writer, "Bucket %s versioning set to %s\n", *bucketName, desiredStatus)
return nil
})
return err
}
func normalizeVersioningStatus(statusFlag string, enable bool, suspend bool) (string, error) {
if enable && suspend {
return "", fmt.Errorf("only one of -enable or -suspend can be set")
}
if (enable || suspend) && statusFlag != "" {
return "", fmt.Errorf("use either -status or -enable/-suspend, not both")
}
if enable {
return s3_constants.VersioningEnabled, nil
}
if suspend {
return s3_constants.VersioningSuspended, nil
}
if statusFlag == "" {
return "", nil
}
switch strings.ToLower(strings.TrimSpace(statusFlag)) {
case "enabled":
return s3_constants.VersioningEnabled, nil
case "suspended":
return s3_constants.VersioningSuspended, nil
default:
return "", fmt.Errorf("invalid versioning status %q: must be Enabled or Suspended", statusFlag)
}
}
func getBucketVersioningState(entry *filer_pb.Entry) (status string, lockEnabled bool) {
if entry.Extended == nil {
return "", false
}
if versioning, ok := entry.Extended[s3_constants.ExtVersioningKey]; ok {
status = string(versioning)
}
if lockStatus, ok := entry.Extended[s3_constants.ExtObjectLockEnabledKey]; ok {
lockEnabled = string(lockStatus) == s3_constants.ObjectLockEnabled
}
return status, lockEnabled
}
Loading…
Cancel
Save