From 6c7fe87a72063178498dcb2319b856e1e6bda229 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 9 Mar 2026 14:24:42 -0700 Subject: [PATCH] helm: add s3.tlsSecret for custom S3 HTTPS certificate (#8582) * helm: add s3.tlsSecret to allow custom TLS certificate for S3 HTTPS endpoint Allow users to specify an external Kubernetes TLS secret for the S3 HTTPS endpoint instead of using the internal self-signed client certificate. This enables using publicly trusted certificates (e.g. from Let's Encrypt) so S3 clients don't need to trust the internal CA. The new s3.tlsSecret value is supported in the standalone S3 gateway, filer with embedded S3, and all-in-one deployment templates. Closes #8581 * refactor: extract S3 TLS helpers to reduce duplication Move repeated S3 TLS cert/key logic into shared helper templates (seaweedfs.s3.tlsArgs, seaweedfs.s3.tlsVolumeMount, seaweedfs.s3.tlsVolume) in _helpers.tpl, and use them across all three deployment templates. * helm: add allInOne.s3.trafficDistribution support Add the missing allInOne.s3.trafficDistribution branch to the seaweedfs.trafficDistribution helper and wire it into the all-in-one service template, mirroring the existing s3-service.yaml behavior. PreferClose is auto-converted to PreferSameZone on k8s >=1.35. * fix: scope S3 TLS mounts to S3-enabled pods and simplify trafficDistribution helper - Wrap S3 TLS volume/volumeMount includes in allInOne.s3.enabled and filer.s3.enabled guards so the custom TLS secret is only mounted when S3 is actually enabled in that deployment mode. - Refactor seaweedfs.trafficDistribution helper to accept an explicit value+Capabilities dict instead of walking multiple .Values paths, making each call site responsible for passing its own setting. --- .../all-in-one/all-in-one-deployment.yaml | 9 ++++- .../all-in-one/all-in-one-service.yml | 3 ++ .../templates/filer/filer-statefulset.yaml | 9 ++++- .../seaweedfs/templates/s3/s3-deployment.yaml | 5 ++- .../seaweedfs/templates/s3/s3-service.yaml | 5 ++- .../seaweedfs/templates/shared/_helpers.tpl | 40 ++++++++++++++++--- k8s/charts/seaweedfs/values.yaml | 8 ++++ 7 files changed, 66 insertions(+), 13 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml index 5a896c12c..b6857c403 100644 --- a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml @@ -243,8 +243,7 @@ spec: {{- if $httpsPort }} -s3.port.https={{ $httpsPort }} \ {{- end }} - -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \ - -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \ + {{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }} {{- end }} {{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }} -s3.config=/etc/sw/s3/seaweedfs_s3_config \ @@ -346,6 +345,9 @@ spec: - name: client-cert mountPath: /usr/local/share/ca-certificates/client/ readOnly: true + {{- if .Values.allInOne.s3.enabled }} + {{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }} + {{- end }} {{- end }} {{ tpl .Values.allInOne.extraVolumeMounts . | nindent 12 }} ports: @@ -473,6 +475,9 @@ spec: - name: client-cert secret: secretName: {{ include "seaweedfs.fullname" . }}-client-cert + {{- if .Values.allInOne.s3.enabled }} + {{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }} + {{- end }} {{- end }} {{ tpl .Values.allInOne.extraVolumes . | nindent 8 }} {{- if .Values.allInOne.nodeSelector }} diff --git a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-service.yml b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-service.yml index 0f1e764bd..90f2f112a 100644 --- a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-service.yml +++ b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-service.yml @@ -17,6 +17,9 @@ metadata: spec: type: {{ .Values.allInOne.service.type | default "ClusterIP" }} internalTrafficPolicy: {{ .Values.allInOne.service.internalTrafficPolicy | default "Cluster" }} + {{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) .Values.allInOne.s3.trafficDistribution }} + trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" .Values.allInOne.s3.trafficDistribution "Capabilities" .Capabilities) }} + {{- end }} ports: # Master ports - name: "swfs-master" diff --git a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml index 7ecf81b45..2aca004b1 100644 --- a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -200,8 +200,7 @@ spec: {{- if .Values.filer.s3.httpsPort }} -s3.port.https={{ .Values.filer.s3.httpsPort }} \ {{- end }} - -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \ - -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \ + {{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }} {{- end }} {{- if .Values.filer.s3.enableAuth }} -s3.config=/etc/sw/seaweedfs_s3_config \ @@ -254,6 +253,9 @@ spec: - name: client-cert readOnly: true mountPath: /usr/local/share/ca-certificates/client + {{- if .Values.filer.s3.enabled }} + {{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }} + {{- end }} {{- end }} {{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }} ports: @@ -384,6 +386,9 @@ spec: - name: client-cert secret: secretName: {{ include "seaweedfs.fullname" . }}-client-cert + {{- if .Values.filer.s3.enabled }} + {{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }} + {{- end }} {{- end }} {{ tpl .Values.filer.extraVolumes . | indent 8 | trim }} {{- if .Values.filer.nodeSelector }} diff --git a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml index 1a83f4406..537419543 100644 --- a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml @@ -127,8 +127,7 @@ spec: {{- if .Values.s3.httpsPort }} -port.https={{ .Values.s3.httpsPort }} \ {{- end }} - -cert.file=/usr/local/share/ca-certificates/client/tls.crt \ - -key.file=/usr/local/share/ca-certificates/client/tls.key \ + {{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }} {{- end }} {{- if .Values.s3.domainName }} -domainName={{ .Values.s3.domainName }} \ @@ -176,6 +175,7 @@ spec: - name: client-cert readOnly: true mountPath: /usr/local/share/ca-certificates/client/ + {{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }} {{- end }} {{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }} ports: @@ -267,6 +267,7 @@ spec: - name: client-cert secret: secretName: {{ include "seaweedfs.fullname" . }}-client-cert + {{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }} {{- end }} {{ tpl .Values.s3.extraVolumes . | indent 8 | trim }} {{- if .Values.s3.nodeSelector }} diff --git a/k8s/charts/seaweedfs/templates/s3/s3-service.yaml b/k8s/charts/seaweedfs/templates/s3/s3-service.yaml index 1230a366d..2471186a1 100644 --- a/k8s/charts/seaweedfs/templates/s3/s3-service.yaml +++ b/k8s/charts/seaweedfs/templates/s3/s3-service.yaml @@ -16,8 +16,9 @@ metadata: {{- end }} spec: internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }} - {{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) (or .Values.s3.trafficDistribution .Values.filer.s3.trafficDistribution) }} - trafficDistribution: {{ include "seaweedfs.trafficDistribution" . }} + {{- $td := .Values.s3.trafficDistribution | default .Values.filer.s3.trafficDistribution }} + {{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) $td }} + trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" $td "Capabilities" .Capabilities) }} {{- end }} ports: - name: "swfs-s3" diff --git a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl index 1ab93efde..52e798b29 100644 --- a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl @@ -338,11 +338,41 @@ Create the name of the service account to use {{- .Values.global.serviceAccountName | default "seaweedfs" -}} {{- end -}} -{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35 */}} +{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}} +{{- define "seaweedfs.s3.tlsArgs" -}} +{{- $prefix := .prefix -}} +{{- $root := .root -}} +{{- if $root.Values.s3.tlsSecret -}} +-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/s3/tls.crt \ +-{{ $prefix }}key.file=/usr/local/share/ca-certificates/s3/tls.key \ +{{- else -}} +-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/client/tls.crt \ +-{{ $prefix }}key.file=/usr/local/share/ca-certificates/client/tls.key \ +{{- end -}} +{{- end -}} + +{{/* S3 custom TLS volume mount */}} +{{- define "seaweedfs.s3.tlsVolumeMount" -}} +{{- if .Values.s3.tlsSecret }} +- name: s3-tls-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/s3/ +{{- end }} +{{- end -}} + +{{/* S3 custom TLS volume */}} +{{- define "seaweedfs.s3.tlsVolume" -}} +{{- if .Values.s3.tlsSecret }} +- name: s3-tls-cert + secret: + secretName: {{ .Values.s3.tlsSecret }} +{{- end }} +{{- end -}} + +{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35. + Accepts a dict with "value" (the trafficDistribution string) and "Capabilities". */}} {{- define "seaweedfs.trafficDistribution" -}} -{{- if .Values.s3.trafficDistribution -}} -{{- and (eq .Values.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.s3.trafficDistribution -}} -{{- else if .Values.filer.s3.trafficDistribution -}} -{{- and (eq .Values.filer.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.filer.s3.trafficDistribution -}} +{{- if .value -}} +{{- and (eq .value "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .value -}} {{- end -}} {{- end -}} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 2a3be6256..49e1e511d 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -919,6 +919,13 @@ s3: port: 8333 # add additional https port httpsPort: 0 + # Use a custom TLS certificate secret for the S3 HTTPS endpoint. + # When set, this Kubernetes Secret (must contain tls.crt and tls.key) is used + # instead of the internal self-signed client certificate generated by cert-manager. + # This allows using a publicly trusted certificate (e.g., from Let's Encrypt) + # so that S3 clients don't need to trust the internal CA. + # Requires global.enableSecurity to be true. + tlsSecret: null metricsPort: 9327 # Iceberg catalog REST port (Apache Iceberg REST Catalog API) # Set to a port number to enable, or 0/null to disable @@ -1453,6 +1460,7 @@ allInOne: # The s3-secret.yaml template only reads from .Values.s3.credentials. # See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig) + trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35 # You may specify buckets to be created during the install process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` # ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)